Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(sessions): import a claude-mem store with --format claude-mem (#2874)

Jacob Elbirt6b77110f48fe
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 78b7e3466..1f3a94856 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -46,6 +46,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
one with `/` is anchored at the root. The run reports how many paths it left
out (#2483).
+- Offline `xerj sessions import --format claude-mem` exports a claude-mem
+ store's prompts, observations and session summaries through the existing
+ text redactor (#2874). It reads `claude-mem.db` read-only in one snapshot,
+ including rows still in a running worker's WAL, joins each row to its Claude
+ Code session UUID, and marks the model-written observations and summaries
+ `is_summary: true`.
+
- Offline `xerj sessions import --format codex` exports a documented subset
of one archived Codex rollout through the existing text redactor. It skips
tools and reasoning, retains physical source locations and archive event
diff --git a/engine/crates/xerj-autoindex/src/sessions.rs b/engine/crates/xerj-autoindex/src/sessions.rs
index 8a3263257..bcbae9602 100644
--- a/engine/crates/xerj-autoindex/src/sessions.rs
+++ b/engine/crates/xerj-autoindex/src/sessions.rs
@@ -16,7 +16,7 @@ const MAX_RECORDS: usize = 100_000;
pub const HELP: &str = "xerj sessions import — offline session preparation for autoindex
-USAGE: xerj sessions import --format <claude-code|codex> --input FILE --output NEW.jsonl
+USAGE: xerj sessions import --format <claude-code|codex|claude-mem> --input FILE --output NEW.jsonl
Read one explicit Claude Code transcript or Codex rollout JSONL; write user/assistant text
records through the existing native PII redactor before any output is stored.
@@ -34,7 +34,10 @@ The redactor handles email/IP/card/SSN/phone patterns, not every secret or name.
Codex reads response_item messages after session_meta, skips duplicate event_msg
text and tools/reasoning; uses line-and-byte-digest event IDs, no inferred parents
or files. exec --json and external history_base are unsupported.
-Input <=512 MiB; line <=1 MiB; turn text <=256 KiB; <=100000 messages.
+claude-mem opens ~/.claude-mem/claude-mem.db read-only in one snapshot and exports
+user_prompts, observations and session_summaries rows (is_summary=true for the
+model-written two), joining each to its Claude Code session UUID via sdk_sessions.
+Input <=512 MiB; line <=1 MiB; turn text <=256 KiB; <=100000 messages or rows.
Exit 0: exported, JSON summary on stdout; 2: usage; 1: import/publication error.
Optional automation (Claude Code, macOS/Linux): xerj init --memory.
@@ -264,6 +267,7 @@ fn claude_turn(
}))
}
+mod claude_mem;
mod codex;
#[derive(Clone, Copy)]
@@ -280,12 +284,95 @@ pub fn import_codex(input: &Path, output: &Path) -> Result<ImportSummary> {
import_session(input, output, SessionFormat::Codex)
}
-fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<ImportSummary> {
+pub fn import_claude_mem(input: &Path, output: &Path) -> Result<ImportSummary> {
+ claude_mem::import(input, output)
+}
+
+fn ensure_new_output(output: &Path) -> Result<()> {
match fs::symlink_metadata(output) {
Ok(_) => bail!("session output already exists; choose a new output path"),
- Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
- Err(e) => return Err(e).context("cannot inspect session output"),
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(e) => Err(e).context("cannot inspect session output"),
+ }
+}
+
+/// Redacted, allowlisted records staged privately until a no-clobber publish.
+struct Export {
+ staging: tempfile::NamedTempFile,
+ redactor: PiiRedactionPlugin,
+ hash: Sha256,
+ written: u64,
+}
+
+impl Export {
+ fn create(output: &Path) -> Result<Self> {
+ let parent = output
+ .parent()
+ .filter(|p| !p.as_os_str().is_empty())
+ .unwrap_or(Path::new("."));
+ let staging = tempfile::Builder::new()
+ .prefix(".xerj-session-")
+ .suffix(".tmp")
+ .tempfile_in(parent)
+ .context("cannot create private export staging file; output parent must exist")?;
+ Ok(Self {
+ staging,
+ redactor: PiiRedactionPlugin::new(Vec::new()),
+ hash: Sha256::new(),
+ written: 0,
+ })
+ }
+
+ fn push(&mut self, turn: ParsedTurn, summary: &mut ImportSummary) -> Result<()> {
+ let ParsedTurn {
+ text_fields: mut record,
+ identity_fields,
+ } = turn;
+ ensure!(
+ self.redactor.process(&mut record) == ProcessAction::Pass,
+ "PII redaction refused session record"
+ );
+ // Only validated enums/IDs and numeric provenance bypass text redaction.
+ record
+ .as_object_mut()
+ .unwrap()
+ .extend(identity_fields.as_object().unwrap().clone());
+ let kind = record["role"].as_str().unwrap();
+ let title: String = record["body"]
+ .as_str()
+ .unwrap()
+ .lines()
+ .next()
+ .unwrap_or("")
+ .chars()
+ .take(120)
+ .collect();
+ record["title"] = json!(format!("{kind}: {title}"));
+ let mut encoded = serde_json::to_vec(&record)?;
+ encoded.push(b'\n');
+ self.written += encoded.len() as u64;
+ ensure!(
+ self.written <= MAX_INPUT_BYTES,
+ "session export exceeds 512 MiB; use a smaller transcript"
+ );
+ self.staging.write_all(&encoded)?;
+ self.hash.update(&encoded);
+ summary.records += 1;
+ Ok(())
+ }
+
+ fn publish(self, output: &Path, summary: &mut ImportSummary) -> Result<()> {
+ let Self {
+ mut staging, hash, ..
+ } = self;
+ summary.output_sha256 = format!("{:x}", hash.finalize());
+ staging.as_file_mut().sync_all()?;
+ publish(staging, output)
}
+}
+
+fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<ImportSummary> {
+ ensure_new_output(output)?;
let mut options = fs::OpenOptions::new();
options.read(true);
#[cfg(unix)]
@@ -305,16 +392,7 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<
.file_name()
.and_then(|s| s.to_str())
.context("session input filename must be UTF-8")?;
- let parent = output
- .parent()
- .filter(|p| !p.as_os_str().is_empty())
- .unwrap_or(Path::new("."));
- let mut staging = tempfile::Builder::new()
- .prefix(".xerj-session-")
- .suffix(".tmp")
- .tempfile_in(parent)
- .context("cannot create private export staging file; output parent must exist")?;
- let redactor = PiiRedactionPlugin::new(Vec::new());
+ let mut export = Export::create(output)?;
let mut summary = ImportSummary {
schema_version: 1,
records: 0,
@@ -325,11 +403,9 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<
output_sha256: String::new(),
};
let mut source_hash = Sha256::new();
- let mut output_hash = Sha256::new();
let mut reader = BufReader::new((&file).take(before.len() + 1));
let mut bytes = Vec::new();
let mut read = 0;
- let mut written = 0u64;
let mut session: Option<String> = None;
let mut seen = HashSet::new();
let mut codex = codex::CodexState::default();
@@ -370,43 +446,9 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<
codex.turn(&entry, line, offset, source_file, &bytes, &mut summary)?
}
};
- let Some(ParsedTurn {
- text_fields: mut record,
- identity_fields,
- }) = parsed
- else {
- continue;
- };
- ensure!(
- redactor.process(&mut record) == ProcessAction::Pass,
- "PII redaction refused session record"
- );
- // Only validated enums/IDs and numeric provenance bypass text redaction.
- record
- .as_object_mut()
- .unwrap()
- .extend(identity_fields.as_object().unwrap().clone());
- let kind = record["role"].as_str().unwrap();
- let title: String = record["body"]
- .as_str()
- .unwrap()
- .lines()
- .next()
- .unwrap_or("")
- .chars()
- .take(120)
- .collect();
- record["title"] = json!(format!("{kind}: {title}"));
- let mut encoded = serde_json::to_vec(&record)?;
- encoded.push(b'\n');
- written += encoded.len() as u64;
- ensure!(
- written <= MAX_INPUT_BYTES,
- "session export exceeds 512 MiB; use a smaller transcript"
- );
- staging.write_all(&encoded)?;
- output_hash.update(&encoded);
- summary.records += 1;
+ if let Some(turn) = parsed {
+ export.push(turn, &mut summary)?;
+ }
}
ensure!(
summary.records > 0,
@@ -414,9 +456,7 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<
);
unchanged(&file, &before, input, read)?;
summary.source_sha256 = format!("{:x}", source_hash.finalize());
- summary.output_sha256 = format!("{:x}", output_hash.finalize());
- staging.as_file_mut().sync_all()?;
- publish(staging, output)?;
+ export.publish(output, &mut summary)?;
Ok(summary)
}
@@ -428,7 +468,8 @@ pub fn run_cli(args: &[String]) -> i32 {
print!("{HELP}");
return 0;
}
- let parsed = (|| -> Result<(PathBuf, PathBuf, SessionFormat)> {
+ type Import = fn(&Path, &Path) -> Result<ImportSummary>;
+ let parsed = (|| -> Result<(PathBuf, PathBuf, Import)> {
ensure!(args[0] == "import", "expected sessions import");
let mut input = None;
let mut output = None;
@@ -448,25 +489,26 @@ pub fn run_cli(args: &[String]) -> i32 {
.context("session import option requires a value")?;
*slot = Some(value.clone());
}
- let format = match format.as_deref() {
- Some("claude-code") => SessionFormat::ClaudeCode,
- Some("codex") => SessionFormat::Codex,
- _ => bail!("--format claude-code or --format codex is required"),
+ let import: Import = match format.as_deref() {
+ Some("claude-code") => import_claude_code,
+ Some("codex") => import_codex,
+ Some("claude-mem") => import_claude_mem,
+ _ => bail!("--format claude-code, codex or claude-mem is required"),
};
Ok((
PathBuf::from(input.context("--input is required")?),
PathBuf::from(output.context("--output is required")?),
- format,
+ import,
))
})();
- let (input, output, format) = match parsed {
+ let (input, output, import) = match parsed {
Ok(paths) => paths,
Err(error) => {
eprintln!("error: {error}");
return 2;
}
};
- match import_session(&input, &output, format) {
+ match import(&input, &output) {
Ok(summary) => {
let mut stdout = std::io::stdout().lock();
if serde_json::to_writer(&mut stdout, &summary).is_err() || writeln!(stdout).is_err() {
diff --git a/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs b/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs
new file mode 100644
index 000000000..a177e5903
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs
@@ -0,0 +1,540 @@
+//! Read-only claude-mem SQLite subset. Reference and licence are pinned in the import guide.
+use super::*;
+use rusqlite::{types::ValueRef, Connection, OpenFlags, Row};
+
+/// One exported claude-mem table: which columns become text, which stay optional.
+struct Table {
+ name: &'static str,
+ role: &'static str,
+ /// Columns every supported claude-mem schema has.
+ required: &'static [&'static str],
+ /// Columns added by later claude-mem migrations; read as NULL when absent.
+ optional: &'static [&'static str],
+ /// Session join: the column that matches `sdk_sessions`, and the column it matches.
+ session: (&'static str, &'static str),
+}
+
+const PROMPTS: Table = Table {
+ name: "user_prompts",
+ role: "user",
+ required: &["id", "prompt_text", "created_at_epoch"],
+ optional: &["prompt_number", "content_session_id"],
+ session: ("content_session_id", "content_session_id"),
+};
+
+const OBSERVATIONS: Table = Table {
+ name: "observations",
+ role: "observation",
+ required: &[
+ "id",
+ "memory_session_id",
+ "project",
+ "type",
+ "created_at_epoch",
+ ],
+ optional: &[
+ "prompt_number",
+ "title",
+ "subtitle",
+ "narrative",
+ "text",
+ "facts",
+ "concepts",
+ "files_read",
+ "files_modified",
+ ],
+ session: ("memory_session_id", "memory_session_id"),
+};
+
+const SUMMARIES: Table = Table {
+ name: "session_summaries",
+ role: "summary",
+ required: &["id", "memory_session_id", "project", "created_at_epoch"],
+ optional: &[
+ "prompt_number",
+ "request",
+ "investigated",
+ "learned",
+ "completed",
+ "next_steps",
+ "notes",
+ "files_read",
+ "files_edited",
+ ],
+ session: ("memory_session_id", "memory_session_id"),
+};
+
+const SUMMARY_SECTIONS: [(&str, &str); 6] = [
+ ("request", "Request"),
+ ("investigated", "Investigated"),
+ ("learned", "Learned"),
+ ("completed", "Completed"),
+ ("next_steps", "Next steps"),
+ ("notes", "Notes"),
+];
+
+fn columns(conn: &Connection, table: &str) -> Result<HashSet<String>> {
+ let mut stmt = conn.prepare("SELECT name FROM pragma_table_info(?1)")?;
+ let names = stmt
+ .query_map([table], |row| row.get::<_, String>(0))?
+ .collect::<rusqlite::Result<HashSet<_>>>()?;
+ Ok(names)
+}
+
+/// A row as JSON: NULL, integer and text only; anything else refuses the export.
+fn row_json(row: &Row, names: &[&str], at: &str) -> Result<serde_json::Map<String, Value>> {
+ let mut out = serde_json::Map::new();
+ for (i, name) in names.iter().enumerate() {
+ let value = match row.get_ref(i)? {
+ ValueRef::Null => Value::Null,
+ ValueRef::Integer(n) => json!(n),
+ ValueRef::Text(t) => json!(std::str::from_utf8(t)
+ .map_err(|_| anyhow::anyhow!("{at}: {name} is not UTF-8 text"))?),
+ _ => bail!("{at}: unsupported value type in {name}"),
+ };
+ out.insert((*name).to_string(), value);
+ }
+ Ok(out)
+}
+
+fn text<'a>(row: &'a serde_json::Map<String, Value>, field: &str, at: &str) -> Result<&'a str> {
+ match &row[field] {
+ Value::Null => Ok(""),
+ Value::String(s) => Ok(s.trim()),
+ _ => bail!("{at}: expected text in {field}"),
+ }
+}
+
+/// claude-mem stores `facts`, `concepts` and `files_*` as JSON.stringify'd string arrays.
+fn string_array(
+ row: &serde_json::Map<String, Value>,
+ field: &str,
+ at: &str,
+) -> Result<Vec<String>> {
+ let raw = text(row, field, at)?;
+ if raw.is_empty() {
+ return Ok(Vec::new());
+ }
+ serde_json::from_str(raw)
+ .map_err(|_| anyhow::anyhow!("{at}: {field} is not a JSON string array"))
+}
+
+fn push_line(body: &mut String, line: &str) {
+ if line.is_empty() {
+ return;
+ }
+ if !body.is_empty() {
+ body.push('\n');
+ }
+ body.push_str(line);
+}
+
+/// Text body and file paths for one row, in the order claude-mem renders them.
+fn body_and_files(
+ table: &Table,
+ row: &serde_json::Map<String, Value>,
+ at: &str,
+) -> Result<(String, Vec<String>)> {
+ let mut body = String::new();
+ let mut files = Vec::new();
+ match table.role {
+ "user" => push_line(&mut body, text(row, "prompt_text", at)?),
+ "observation" => {
+ for field in ["title", "subtitle", "narrative", "text"] {
+ push_line(&mut body, text(row, field, at)?);
+ }
+ for fact in string_array(row, "facts", at)? {
+ push_line(&mut body, &format!("- {}", fact.trim()));
+ }
+ files.extend(string_array(row, "files_modified", at)?);
+ files.extend(string_array(row, "files_read", at)?);
+ }
+ _ => {
+ for (field, label) in SUMMARY_SECTIONS {
+ let value = text(row, field, at)?;
+ if !value.is_empty() {
+ push_line(&mut body, &format!("{label}: {value}"));
+ }
+ }
+ files.extend(string_array(row, "files_edited", at)?);
+ files.extend(string_array(row, "files_read", at)?);
+ }
+ }
+ ensure!(body.len() <= MAX_TEXT_BYTES, "{at}: text exceeds 256 KiB");
+ let mut unique = HashSet::new();
+ files.retain(|f| unique.insert(f.clone()));
+ Ok((body, files))
+}
+
+fn timestamp(row: &serde_json::Map<String, Value>, at: &str) -> Result<Value> {
+ let millis = row["created_at_epoch"]
+ .as_i64()
+ .with_context(|| format!("{at}: expected integer created_at_epoch"))?;
+ let time = chrono::DateTime::from_timestamp_millis(millis)
+ .with_context(|| format!("{at}: created_at_epoch out of range"))?;
+ Ok(json!(
+ time.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
+ ))
+}
+
+#[allow(clippy::too_many_arguments)]
+fn export_table(
+ conn: &Connection,
+ table: &Table,
+ sessions: bool,
+ source_file: &str,
+ summary: &mut ImportSummary,
+ source_hash: &mut Sha256,
+ export: &mut Export,
+) -> Result<()> {
+ let present = columns(conn, table.name)?;
+ if present.is_empty() {
+ return Ok(());
+ }
+ for name in table.required {
+ ensure!(
+ present.contains(*name),
+ "claude-mem table {} has no {name} column; unsupported claude-mem version",
+ table.name
+ );
+ }
+ let names: Vec<&str> = table
+ .required
+ .iter()
+ .chain(table.optional)
+ .copied()
+ .collect();
+ // Column and table names come only from the allowlists above.
+ let mut select: Vec<String> = names
+ .iter()
+ .map(|c| match present.contains(*c) {
+ true => format!("t.{c}"),
+ false => format!("NULL AS {c}"),
+ })
+ .collect();
+ let (local, remote) = table.session;
+ let join = if sessions && present.contains(local) {
+ select.push("s.content_session_id".to_string());
+ format!("LEFT JOIN sdk_sessions s ON s.{remote} = t.{local}")
+ } else {
+ select.push("NULL".to_string());
+ String::new()
+ };
+ let sql = format!(
+ "SELECT {} FROM {} t {join} ORDER BY t.created_at_epoch, t.id",
+ select.join(", "),
+ table.name
+ );
+ let mut stmt = conn.prepare(&sql)?;
+ let mut rows = stmt.query([])?;
+ let mut all = names.clone();
+ all.push("session");
+ while let Some(row) = rows.next()? {
+ summary.source_lines += 1;
+ ensure!(
+ summary.source_lines <= MAX_RECORDS as u64,
+ "claude-mem store exceeds 100000 rows; export a smaller copy"
+ );
+ let at = format!("{} row {}", table.name, row.get::<_, i64>(0)?);
+ let row = row_json(row, &all, &at)?;
+ source_hash.update(table.name.as_bytes());
+ source_hash.update(serde_json::to_vec(&row)?);
+ let (body, files) = body_and_files(table, &row, &at)?;
+ if body.is_empty() {
+ summary.skipped_records += 1;
+ continue;
+ }
+ let rowid = row["id"].as_i64().unwrap();
+ let mut text_fields = json!({"body": body, "files": files, "source_file": source_file});
+ for field in ["project", "memory_session_id"] {
+ if let Some(value) = row.get(field).filter(|v| !v.is_null()) {
+ text_fields[field] = value.clone();
+ }
+ }
+ if table.role == "observation" {
+ text_fields["observation_type"] = row["type"].clone();
+ text_fields["concepts"] = json!(string_array(&row, "concepts", &at)?);
+ }
+ let mut record = json!({});
+ record["id"] = json!(format!("claude-mem/{}/{rowid}", table.role));
+ record["session_schema_version"] = json!(1);
+ record["session_format"] = json!("claude-mem");
+ // claude-mem keys sessions by its own ID; the Claude Code session UUID is joined in.
+ let session = row["session"]
+ .as_str()
+ .or_else(|| row.get("content_session_id").and_then(Value::as_str));
+ record["session_id"] = Value::Null;
+ if let Some(session) = session {
+ match uuid::Uuid::parse_str(session) {
+ Ok(id) if session.len() == 36 => record["session_id"] = json!(id.to_string()),
+ // Not a UUID (another claude-mem platform): keep it, but as redacted text.
+ _ => text_fields["session_id"] = json!(session),
+ }
+ }
+ record["message_id"] = json!(format!("{}/{rowid}", table.role));
+ record["parent_id"] = Value::Null;
+ record["role"] = json!(table.role);
+ record["timestamp"] = timestamp(&row, &at)?;
+ record["source_table"] = json!(table.name);
+ record["source_line"] = json!(rowid);
+ record["prompt_number"] = row
+ .get("prompt_number")
+ .filter(|v| v.is_i64())
+ .cloned()
+ .unwrap_or(Value::Null);
+ // Observations and summaries are claude-mem's model-written condensations.
+ record["is_summary"] = json!(table.role != "user");
+ export.push(
+ ParsedTurn {
+ text_fields,
+ identity_fields: record,
+ },
+ summary,
+ )?;
+ }
+ Ok(())
+}
+
+pub(super) fn import(input: &Path, output: &Path) -> Result<ImportSummary> {
+ ensure_new_output(output)?;
+ let meta = fs::metadata(input).context("cannot open claude-mem database")?;
+ ensure!(meta.is_file(), "claude-mem input must be a regular file");
+ ensure!(
+ meta.len() <= MAX_INPUT_BYTES,
+ "claude-mem database exceeds 512 MiB; export a smaller copy"
+ );
+ let source_file = input
+ .file_name()
+ .and_then(|s| s.to_str())
+ .context("claude-mem input filename must be UTF-8")?;
+ // Read-only but not immutable: a running claude-mem worker keeps recent rows in the WAL.
+ let conn = Connection::open_with_flags(
+ input,
+ OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
+ )
+ .context("cannot open claude-mem database read-only")?;
+ conn.execute_batch("PRAGMA query_only = 1; BEGIN")
+ .context("cannot read claude-mem database")?;
+ let tables = columns(&conn, "observations").map_err(|_| {
+ anyhow::anyhow!("input is not a readable SQLite database; pass ~/.claude-mem/claude-mem.db")
+ })?;
+ ensure!(
+ !tables.is_empty(),
+ "not a claude-mem database: no observations table"
+ );
+ let sessions = !columns(&conn, "sdk_sessions")?.is_empty();
+ let mut export = Export::create(output)?;
+ let mut summary = ImportSummary {
+ schema_version: 1,
+ records: 0,
+ source_lines: 0,
+ skipped_records: 0,
+ skipped_blocks: 0,
+ source_sha256: String::new(),
+ output_sha256: String::new(),
+ };
+ let mut source_hash = Sha256::new();
+ // One read transaction: all three tables come from the same snapshot.
+ for table in [&PROMPTS, &OBSERVATIONS, &SUMMARIES] {
+ export_table(
+ &conn,
+ table,
+ sessions,
+ source_file,
+ &mut summary,
+ &mut source_hash,
+ &mut export,
+ )?;
+ }
+ conn.execute_batch("COMMIT")?;
+ ensure!(
+ summary.records > 0,
+ "no text-bearing claude-mem prompts, observations or summaries; no export published"
+ );
+ summary.source_sha256 = format!("{:x}", source_hash.finalize());
+ export.publish(output, &mut summary)?;
+ Ok(summary)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn store(dir: &Path) -> PathBuf {
+ let path = dir.join("claude-mem.db");
+ Connection::open(&path)
+ .unwrap()
+ .execute_batch(include_str!("../../tests/fixtures/claude-mem.sql"))
+ .unwrap();
+ path
+ }
+
+ fn rows(path: &Path) -> Vec<Value> {
+ fs::read_to_string(path)
+ .unwrap()
+ .lines()
+ .map(|s| serde_json::from_str(s).unwrap())
+ .collect()
+ }
+
+ #[test]
+ fn claude_mem_prompts_observations_and_summaries_export_redacted_with_sessions() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = store(dir.path());
+ let original = fs::read(&input).unwrap();
+ let output = dir.path().join("safe.jsonl");
+ let summary = import_claude_mem(&input, &output).unwrap();
+ assert_eq!((summary.records, summary.source_lines), (3, 3));
+ let safe = fs::read_to_string(&output).unwrap();
+ assert!(!safe.contains("DO_NOT_EXPORT"));
+ assert!(!safe.contains("[email protected]"));
+ assert!(safe.contains("[REDACTED_EMAIL]"));
+ let rows = rows(&output);
+ let roles: Vec<_> = rows.iter().map(|r| r["role"].as_str().unwrap()).collect();
+ assert_eq!(roles, ["user", "observation", "summary"]);
+ for row in &rows {
+ assert_eq!(row["session_format"], "claude-mem");
+ assert_eq!(row["session_id"], "550e8400-e29b-41d4-a716-446655440000");
+ }
+ let observation = &rows[1];
+ assert_eq!(observation["id"], "claude-mem/observation/1");
+ assert_eq!(
+ observation["title"],
+ "observation: Dropped polling for file events"
+ );
+ assert!(observation["body"]
+ .as_str()
+ .unwrap()
+ .contains("- The durable journal preserves recovery"));
+ assert_eq!(observation["files"], json!(["src/watch.rs", "src/poll.rs"]));
+ assert_eq!(observation["observation_type"], "decision");
+ assert_eq!(observation["concepts"], json!(["watcher", "recovery"]));
+ assert_eq!(observation["source_table"], "observations");
+ assert_eq!(observation["source_line"], 1);
+ assert_eq!(observation["timestamp"], "2026-10-25T00:00:01.000Z");
+ assert_eq!(
+ [
+ &rows[0]["is_summary"],
+ &rows[1]["is_summary"],
+ &rows[2]["is_summary"]
+ ],
+ [false, true, true]
+ );
+ assert!(rows[2]["body"]
+ .as_str()
+ .unwrap()
+ .starts_with("Request: Explain why polling was dropped\nInvestigated:"));
+ assert_eq!(rows[2]["files"], json!(["src/watch.rs", "src/poll.rs"]));
+ // Same snapshot, same bytes; the store itself is never written.
+ let again = dir.path().join("again.jsonl");
+ let second = import_claude_mem(&input, &again).unwrap();
+ assert_eq!(fs::read_to_string(&again).unwrap(), safe);
+ assert_eq!(second.source_sha256, summary.source_sha256);
+ assert_eq!(fs::read(&input).unwrap(), original);
+ assert!(import_claude_mem(&input, &output).is_err());
+ assert_eq!(fs::read_to_string(&output).unwrap(), safe);
+ }
+
+ #[test]
+ fn claude_mem_rows_still_in_a_live_workers_wal_are_exported() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = store(dir.path());
+ // A running worker: WAL mode, no checkpoint, connection held open.
+ let worker = Connection::open(&input).unwrap();
+ worker
+ .execute_batch(
+ "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;
+ INSERT INTO observations (memory_session_id, project, type, created_at,
+ created_at_epoch, title) VALUES ('mem-0001', 'watcher', 'discovery',
+ '2026-10-25T00:00:03.000Z', 1792886403000, 'Written after the snapshot');",
+ )
+ .unwrap();
+ assert!(dir.path().join("claude-mem.db-wal").exists());
+ let output = dir.path().join("live.jsonl");
+ let summary = import_claude_mem(&input, &output).unwrap();
+ assert_eq!(summary.records, 4);
+ assert!(fs::read_to_string(&output)
+ .unwrap()
+ .contains("Written after the snapshot"));
+ drop(worker);
+ }
+
+ #[test]
+ fn claude_mem_older_schema_without_migrated_columns_still_exports() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("claude-mem.db");
+ Connection::open(&input)
+ .unwrap()
+ .execute_batch(
+ "CREATE TABLE observations (id INTEGER PRIMARY KEY, memory_session_id TEXT,
+ project TEXT, text TEXT, type TEXT, created_at TEXT, created_at_epoch INTEGER);
+ INSERT INTO observations VALUES (7, 'mem-1', 'p', 'Plain v1 text', 'change',
+ '2026-10-25T00:00:00.000Z', 1792886400000);",
+ )
+ .unwrap();
+ let output = dir.path().join("safe.jsonl");
+ let summary = import_claude_mem(&input, &output).unwrap();
+ assert_eq!(summary.records, 1);
+ let rows = rows(&output);
+ assert_eq!(rows[0]["body"], "Plain v1 text");
+ assert_eq!(rows[0]["session_id"], Value::Null);
+ assert_eq!(rows[0]["files"], json!([]));
+ }
+
+ #[test]
+ fn claude_mem_wrong_or_malformed_stores_publish_nothing() {
+ let dir = tempfile::tempdir().unwrap();
+ let output = dir.path().join("safe.jsonl");
+ let not_sqlite = dir.path().join("notes.db");
+ fs::write(&not_sqlite, "not a database").unwrap();
+ let other = dir.path().join("other.db");
+ Connection::open(&other)
+ .unwrap()
+ .execute_batch("CREATE TABLE t (x)")
+ .unwrap();
+ let malformed = store(dir.path());
+ Connection::open(&malformed)
+ .unwrap()
+ .execute_batch("UPDATE observations SET facts = 'not json'")
+ .unwrap();
+ for (input, reason) in [
+ (&not_sqlite, "not a readable SQLite database"),
+ (&other, "no observations table"),
+ (
+ &malformed,
+ "observations row 1: facts is not a JSON string array",
+ ),
+ ] {
+ let error = format!("{:#}", import_claude_mem(input, &output).unwrap_err());
+ assert!(error.contains(reason), "{error}");
+ assert!(!output.exists());
+ }
+ assert_eq!(
+ fs::read_dir(dir.path())
+ .unwrap()
+ .filter(|e| e
+ .as_ref()
+ .unwrap()
+ .file_name()
+ .to_string_lossy()
+ .ends_with(".tmp"))
+ .count(),
+ 0
+ );
+ }
+
+ #[test]
+ fn claude_mem_is_a_cli_format() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = store(dir.path());
+ let output = dir.path().join("safe.jsonl");
+ let args: Vec<String> = ["import", "--format", "claude-mem", "--input"]
+ .iter()
+ .map(|s| s.to_string())
+ .chain([input.display().to_string(), "--output".into()])
+ .chain([output.display().to_string()])
+ .collect();
+ assert_eq!(run_cli(&args), 0);
+ assert_eq!(rows(&output).len(), 3);
+ }
+}
diff --git a/engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sql b/engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sql
new file mode 100644
index 000000000..fa0d80c03
--- /dev/null
+++ b/engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sql
@@ -0,0 +1,97 @@
+-- Synthetic claude-mem store: the four tables `xerj sessions import --format claude-mem`
+-- reads, in the column shape of thedotmack/claude-mem at 999a0c40 (SessionStore.ts).
+-- Columns the importer never exports carry DO_NOT_EXPORT markers.
+PRAGMA journal_mode = WAL;
+
+CREATE TABLE sdk_sessions (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ content_session_id TEXT NOT NULL,
+ memory_session_id TEXT UNIQUE,
+ project TEXT NOT NULL,
+ platform_source TEXT NOT NULL DEFAULT 'claude',
+ user_prompt TEXT,
+ started_at TEXT NOT NULL,
+ started_at_epoch INTEGER NOT NULL,
+ status TEXT NOT NULL DEFAULT 'active',
+ custom_title TEXT,
+ cwd TEXT
+);
+
+CREATE TABLE observations (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ memory_session_id TEXT NOT NULL,
+ project TEXT NOT NULL,
+ text TEXT,
+ type TEXT NOT NULL,
+ created_at TEXT NOT NULL,
+ created_at_epoch INTEGER NOT NULL,
+ prompt_number INTEGER,
+ title TEXT,
+ subtitle TEXT,
+ facts TEXT,
+ narrative TEXT,
+ concepts TEXT,
+ files_read TEXT,
+ files_modified TEXT,
+ discovery_tokens INTEGER DEFAULT 0,
+ content_hash TEXT,
+ merged_into_project TEXT,
+ metadata TEXT
+);
+
+CREATE TABLE session_summaries (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ memory_session_id TEXT UNIQUE NOT NULL,
+ project TEXT NOT NULL,
+ request TEXT,
+ investigated TEXT,
+ learned TEXT,
+ completed TEXT,
+ next_steps TEXT,
+ files_read TEXT,
+ files_edited TEXT,
+ notes TEXT,
+ created_at TEXT NOT NULL,
+ created_at_epoch INTEGER NOT NULL,
+ prompt_number INTEGER,
+ discovery_tokens INTEGER DEFAULT 0
+);
+
+CREATE TABLE user_prompts (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ session_db_id INTEGER,
+ content_session_id TEXT NOT NULL,
+ prompt_number INTEGER NOT NULL,
+ prompt_text TEXT NOT NULL,
+ created_at TEXT NOT NULL,
+ created_at_epoch INTEGER NOT NULL
+);
+
+INSERT INTO sdk_sessions VALUES (
+ 1, '550e8400-e29b-41d4-a716-446655440000', 'mem-0001', 'watcher', 'claude',
+ 'DO_NOT_EXPORT_SESSION_PROMPT', '2026-10-25T00:00:00.000Z', 1792886400000,
+ 'completed', 'DO_NOT_EXPORT_TITLE', '/work/watcher'
+);
+
+INSERT INTO user_prompts VALUES (
+ 1, 1, '550e8400-e29b-41d4-a716-446655440000', 1,
+ 'Why did we drop polling? Contact [email protected] or 202-555-0147.',
+ '2026-10-25T00:00:00.000Z', 1792886400000
+);
+
+INSERT INTO observations VALUES (
+ 1, 'mem-0001', 'watcher', NULL, 'decision', '2026-10-25T00:00:01.000Z', 1792886401000, 1,
+ 'Dropped polling for file events', 'Watcher design',
+ '["File events avoid repeated scans","The durable journal preserves recovery"]',
+ 'We dropped polling because file events avoid repeated scans while the durable journal preserves recovery.',
+ '["watcher","recovery"]', '["src/poll.rs"]', '["src/watch.rs"]', 1200,
+ 'DO_NOT_EXPORT_HASH', NULL, '{"note":"DO_NOT_EXPORT_METADATA"}'
+);
+
+INSERT INTO session_summaries VALUES (
+ 1, 'mem-0001', 'watcher', 'Explain why polling was dropped',
+ 'Compared polling with file events', 'Event loss is recoverable from the journal',
+ 'Switched the watcher to file events', 'Measure event latency',
+ '["src/poll.rs"]', '["src/watch.rs"]', NULL,
+ '2026-10-25T00:00:02.000Z', 1792886402000, 1, 900
+);
diff --git a/tools/session-import/README.md b/tools/session-import/README.md
index e5b14aa0e..4d778a202 100644
--- a/tools/session-import/README.md
+++ b/tools/session-import/README.md
@@ -1,10 +1,11 @@
# Import an archived agent transcript for decision retrieval
`xerj sessions import` is the explicit preparation step for a single Claude Code
-transcript or Codex rollout JSONL file. It runs locally, calls no model or node, and exports text
+transcript, Codex rollout JSONL file or claude-mem store. It runs locally, calls no model or node, and exports text
turns through XERJ's existing native PII plugin. Review the export, then give its
-folder to ordinary `xerj autoindex`. This covers two explicit formats for
-[#1793](https://github.com/xerj-org/xerj/issues/1793), not automatic transcript
+folder to ordinary `xerj autoindex`. This covers three explicit formats for
+[#1793](https://github.com/xerj-org/xerj/issues/1793) and
+[#2874](https://github.com/xerj-org/xerj/issues/2874), not automatic transcript
discovery, a live conversation view or incremental sync.
## Try the synthetic example
@@ -231,3 +232,80 @@ Run the Codex fixture through the same private-node lifecycle check:
```sh
python3 tools/session-import/smoke.py --binary engine/target/release/xerj --format codex
```
+
+## claude-mem store adapter
+
+[claude-mem](https://github.com/thedotmack/claude-mem) is a Claude Code plugin
+that has a model condense each session into observations and summaries, kept in
+one SQLite file. Pass that file explicitly; the command does not look for it,
+start or stop the claude-mem worker, or write to the store:
+
+```sh
+set -eu
+SESSION_EXPORT=$(mktemp -d)
+xerj sessions import --format claude-mem \
+ --input ~/.claude-mem/claude-mem.db \
+ --output "$SESSION_EXPORT/records.jsonl"
+```
+
+The store is opened read-only, not immutable, inside one read transaction, so
+rows a running worker still holds in `claude-mem.db-wal` are included and all
+three tables come from the same snapshot. SQLite needs the store's `-shm` file
+or a writable folder to read a WAL database; on a read-only copy without either,
+copy the `.db`, `-wal` and `-shm` files together. A file with no `observations`
+table refuses as not a claude-mem store.
+
+Three tables export, each ordered by `created_at_epoch` then row id:
+
+| Table | `role` | `body` |
+|---|---|---|
+| `user_prompts` | `user` | `prompt_text`, what was typed |
+| `observations` | `observation` | `title`, `subtitle`, `narrative`, `text`, then each of `facts` as a `- ` line |
+| `session_summaries` | `summary` | `Request:`, `Investigated:`, `Learned:`, `Completed:`, `Next steps:`, `Notes:` lines |
+
+Observations and summaries are what claude-mem's model wrote about a session,
+not what was said in it: they carry `is_summary: true`, and a search hit on one
+is a model's account to check against the code, not a quote. A row whose text
+columns are all empty counts as a skipped record. Session metadata (the stored
+first prompt and custom title), content hashes, token counts, the `metadata`
+column, `pending_messages` and the Chroma vector store are not exported.
+
+claude-mem output keeps schema 1 and the shared text/source fields, with these
+format-specific meanings:
+
+| Field | claude-mem meaning |
+|---|---|
+| `session_format`, `session_id` | `claude-mem`; the Claude Code session UUID (`content_session_id`), joined through `sdk_sessions` for observations and summaries. A non-UUID value is kept as redacted text; no session row gives null |
+| `id`, `message_id` | `claude-mem/<role>/<row id>` and `<role>/<row id>`: stable for a row across exports of the same store |
+| `parent_id` | null |
+| `files` | `files_modified` (summaries: `files_edited`) then `files_read`, deduplicated, as claude-mem recorded them |
+| `project`, `memory_session_id` | claude-mem's project name and its own session key, as redacted text |
+| `observation_type`, `concepts` | Observations only: claude-mem's type (`decision`, `bugfix` and so on) and concept tags, as redacted text |
+| `prompt_number` | The prompt within the session the row belongs to, or null |
+| `timestamp` | `created_at_epoch` as RFC 3339 UTC with milliseconds |
+| `source_file`, `source_table`, `source_line` | Store basename, table, and SQLite row id (there is no line or byte offset in a database) |
+
+`source_lines` in the summary counts rows read and `source_sha256` hashes those
+rows' selected columns in export order, so two exports of an unchanged store
+match. Columns later claude-mem migrations added are read as null on an older
+store; a missing required column, a value that is not text or an integer, or a
+`facts`, `concepts` or `files_*` value that is not a JSON string array refuses
+the whole export. More than 100,000 rows refuses rather than truncating. The
+redaction, size limits, staging, no-clobber publication, exit codes and
+new-folder/prefix rerun rules above apply unchanged.
+
+Reference retrieved before implementation: claude-mem, Apache-2.0, pinned at
+`999a0c4010a12befecb1686e695126c04a4faa6e`:
+
+- [`SessionStore.ts:1273-1325`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L1273-L1325): base `sdk_sessions`, `observations` and `session_summaries` tables and their session keys.
+- [`SessionStore.ts:1697-1712`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L1697-L1712): `user_prompts`, keyed by `content_session_id` and `session_db_id`, with no project column.
+- [`SessionStore.ts:4166-4180`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L4166-L4180): observation fields, with `facts`, `concepts` and `files_*` stored as `JSON.stringify` arrays.
+
+These are format evidence, not copied implementation. The fixture
+(`engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sql`) is a synthetic
+store in that column shape, not a live claude-mem install. Run it through the
+private-node lifecycle check:
+
+```sh
+python3 tools/session-import/smoke.py --binary engine/target/release/xerj --format claude-mem
+```
diff --git a/tools/session-import/smoke.py b/tools/session-import/smoke.py
index b5bdce844..20df4a2db 100644
--- a/tools/session-import/smoke.py
+++ b/tools/session-import/smoke.py
@@ -1,16 +1,24 @@
#!/usr/bin/env python3
-"""Synthetic session -> redacted JSONL -> autoindex -> cited decision, on a private node."""
+"""Synthetic session or claude-mem store -> redacted JSONL -> autoindex -> cited decision, on a private node."""
import argparse
import json
import os
from pathlib import Path
import socket
+import sqlite3
import subprocess
import tempfile
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[2]
+FIXTURES = ROOT / 'engine/crates/xerj-autoindex/tests/fixtures'
+# fixture, records, decision source_line, decision files
+FORMATS = {
+ 'claude-code': ('claude-session.jsonl', 2, 3, ['src/watch.rs']),
+ 'codex': ('codex-session.jsonl', 2, 9, []),
+ 'claude-mem': ('claude-mem.sql', 3, 1, ['src/watch.rs', 'src/poll.rs']),
+}
def run(binary, arguments, env, work, expected=0):
@@ -24,7 +32,7 @@ def run(binary, arguments, env, work, expected=0):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--binary', type=Path, required=True)
- parser.add_argument('--format', choices=('claude-code', 'codex'), default='claude-code')
+ parser.add_argument('--format', choices=tuple(FORMATS), default='claude-code')
args = parser.parse_args()
binary = args.binary.resolve(strict=True)
env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')
@@ -32,20 +40,26 @@ def main():
env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_AUTH='', XERJ_NUM_CPUS='4')
with tempfile.TemporaryDirectory(prefix='xerj-sessions-smoke-') as temp:
work = Path(temp)
- source = work / 'session.jsonl'
- fixture = 'codex-session.jsonl' if args.format == 'codex' else 'claude-session.jsonl'
- source.write_bytes((ROOT / 'engine/crates/xerj-autoindex/tests/fixtures' / fixture).read_bytes())
+ fixture, records, expected_line, expected_files = FORMATS[args.format]
+ if args.format == 'claude-mem':
+ source = work / 'claude-mem.db'
+ with sqlite3.connect(source) as store:
+ store.executescript((FIXTURES / fixture).read_text())
+ store.close()
+ else:
+ source = work / 'session.jsonl'
+ source.write_bytes((FIXTURES / fixture).read_bytes())
exports = work / 'export'
exports.mkdir()
output = exports / 'records.jsonl'
command = ['sessions', 'import', '--format', args.format, '--input', str(source), '--output', str(output)]
imported = json.loads(run(binary, command, env, work).stdout)
- assert imported['records'] == 2, imported
+ assert imported['records'] == records, imported
safe = output.read_bytes()
assert b'[email protected]' not in safe and b'DO_NOT_EXPORT' not in safe
run(binary, command, env, work, expected=1)
assert output.read_bytes() == safe
- print('offline import: 2 redacted text turns; overwrite refused', flush=True)
+ print(f'offline import: {records} redacted text records; overwrite refused', flush=True)
ports = []
sockets = []
@@ -128,29 +142,27 @@ def main():
assert len(done) == 1 and done[0]['ok'] is True and done[0]['exit'] == 0, done
assert not done[0].get('datasets_refused') and not done[0].get('files_refused'), done
count = request('/session-smoke-*/_count')['count']
- assert count == 2, count
+ assert count == records, count
print(f'autoindex pass {generation + 1}: {time.monotonic() - before:.3f}s; {count} live documents', flush=True)
result = request('/session-smoke-*/_search', {'size': 10, 'query': {'match_all': {}}})
stored = result['hits']['hits']
- assert len(stored) == 2, result
+ assert len(stored) == records, result
assert '[email protected]' not in json.dumps(stored)
assert 'DO_NOT_EXPORT' not in json.dumps(stored)
decision = next(h['_source'] for h in stored if 'durable journal' in h['_source']['body'])
- expected_line = 9 if args.format == 'codex' else 3
- expected_files = [] if args.format == 'codex' else ['src/watch.rs']
- assert decision['source_file'] == 'session.jsonl' and decision['source_line'] == expected_line, decision
+ assert decision['source_file'] == source.name and decision['source_line'] == expected_line, decision
assert decision['files'] == expected_files, decision
assert decision['session_format'] == args.format, decision
search = run(binary, ['search', '--prefix', 'session-smoke', '--url', url,
'--json', 'why did we drop polling'], env, work)
assert 'durable journal' in search.stdout, search.stdout
- print(f'search: decision text found; original session.jsonl:{expected_line}; files={expected_files}', flush=True)
+ print(f'search: decision text found; original {source.name}:{expected_line}; files={expected_files}', flush=True)
finally:
stop(child)
child = start(log)
try:
- assert request('/session-smoke-*/_count')['count'] == 2
- print('graceful restart: 2 documents preserved', flush=True)
+ assert request('/session-smoke-*/_count')['count'] == records
+ print(f'graceful restart: {records} documents preserved', flush=True)
finally:
stop(child)
return 0