Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(sessions): keep non-UUID session ids and export each claude-mem prompt once

Jacob Elbirtecc2e2672608
diff --git a/engine/crates/xerj-autoindex/src/sessions.rs b/engine/crates/xerj-autoindex/src/sessions.rs
index bcbae9602..04402f598 100644
--- a/engine/crates/xerj-autoindex/src/sessions.rs
+++ b/engine/crates/xerj-autoindex/src/sessions.rs
@@ -36,8 +36,9 @@ text and tools/reasoning; uses line-and-byte-digest event IDs, no inferred paren
or files. exec --json and external history_base are unsupported.
claude-mem opens ~/.claude-mem/claude-mem.db read-only in one snapshot and exports
user_prompts, observations and session_summaries rows (is_summary=true for the
-model-written two), joining each to its Claude Code session UUID via sdk_sessions.
-Input <=512 MiB; line <=1 MiB; turn text <=256 KiB; <=100000 messages or rows.
+model-written two), each with the session ID claude-mem recorded (a UUID for
+Claude Code). Input <=512 MiB (.db plus -wal); line <=1 MiB; turn text
+<=256 KiB; <=100000 messages or rows.
Exit 0: exported, JSON summary on stdout; 2: usage; 1: import/publication error.
Optional automation (Claude Code, macOS/Linux): xerj init --memory.
diff --git a/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs b/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs
index a177e5903..c57beec73 100644
--- a/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs
+++ b/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs
@@ -10,8 +10,8 @@ struct Table {
required: &'static [&'static str],
/// Columns added by later claude-mem migrations; read as NULL when absent.
optional: &'static [&'static str],
- /// Session join: the column that matches `sdk_sessions`, and the column it matches.
- session: (&'static str, &'static str),
+ /// Joins `sdk_sessions` on this column (unique there) for the platform session ID.
+ session: Option<&'static str>,
}
const PROMPTS: Table = Table {
@@ -19,7 +19,8 @@ const PROMPTS: Table = Table {
role: "user",
required: &["id", "prompt_text", "created_at_epoch"],
optional: &["prompt_number", "content_session_id"],
- session: ("content_session_id", "content_session_id"),
+ // The row carries content_session_id itself; sdk_sessions repeats it once per platform.
+ session: None,
};
const OBSERVATIONS: Table = Table {
@@ -43,7 +44,7 @@ const OBSERVATIONS: Table = Table {
"files_read",
"files_modified",
],
- session: ("memory_session_id", "memory_session_id"),
+ session: Some("memory_session_id"),
};
const SUMMARIES: Table = Table {
@@ -61,7 +62,7 @@ const SUMMARIES: Table = Table {
"files_read",
"files_edited",
],
- session: ("memory_session_id", "memory_session_id"),
+ session: Some("memory_session_id"),
};
const SUMMARY_SECTIONS: [(&str, &str); 6] = [
@@ -177,7 +178,6 @@ fn timestamp(row: &serde_json::Map<String, Value>, at: &str) -> Result<Value> {
))
}
-#[allow(clippy::too_many_arguments)]
fn export_table(
conn: &Connection,
table: &Table,
@@ -191,6 +191,12 @@ fn export_table(
if present.is_empty() {
return Ok(());
}
+ ensure!(
+ !present.contains("sdk_session_id"),
+ "claude-mem table {} predates claude-mem's session ID rename; start claude-mem once to \
+ migrate the store, then rerun",
+ table.name
+ );
for name in table.required {
ensure!(
present.contains(*name),
@@ -207,15 +213,17 @@ fn export_table(
// Column and table names come only from the allowlists above.
let mut select: Vec<String> = names
.iter()
- .map(|c| match present.contains(*c) {
- true => format!("t.{c}"),
- false => format!("NULL AS {c}"),
+ .map(|c| {
+ if present.contains(*c) {
+ format!("t.{c}")
+ } else {
+ format!("NULL AS {c}")
+ }
})
.collect();
- let (local, remote) = table.session;
- let join = if sessions && present.contains(local) {
+ let join = if let Some(key) = table.session.filter(|_| sessions) {
select.push("s.content_session_id".to_string());
- format!("LEFT JOIN sdk_sessions s ON s.{remote} = t.{local}")
+ format!("LEFT JOIN sdk_sessions s ON s.{key} = t.{key}")
} else {
select.push("NULL".to_string());
String::new()
@@ -259,17 +267,15 @@ fn export_table(
record["id"] = json!(format!("claude-mem/{}/{rowid}", table.role));
record["session_schema_version"] = json!(1);
record["session_format"] = json!("claude-mem");
- // claude-mem keys sessions by its own ID; the Claude Code session UUID is joined in.
+ // claude-mem keys sessions by its own ID; the platform's session ID is joined in.
let session = row["session"]
.as_str()
.or_else(|| row.get("content_session_id").and_then(Value::as_str));
- record["session_id"] = Value::Null;
- if let Some(session) = session {
- match uuid::Uuid::parse_str(session) {
- Ok(id) if session.len() == 36 => record["session_id"] = json!(id.to_string()),
- // Not a UUID (another claude-mem platform): keep it, but as redacted text.
- _ => text_fields["session_id"] = json!(session),
- }
+ match session.map(|s| (s, uuid::Uuid::parse_str(s))) {
+ Some((s, Ok(id))) if s.len() == 36 => record["session_id"] = json!(id.to_string()),
+ // Not a UUID (another claude-mem platform): keep it, but as redacted text.
+ Some((s, _)) => text_fields["session_id"] = json!(s),
+ None => record["session_id"] = Value::Null,
}
record["message_id"] = json!(format!("{}/{rowid}", table.role));
record["parent_id"] = Value::Null;
@@ -299,8 +305,12 @@ pub(super) fn import(input: &Path, output: &Path) -> Result<ImportSummary> {
ensure_new_output(output)?;
let meta = fs::metadata(input).context("cannot open claude-mem database")?;
ensure!(meta.is_file(), "claude-mem input must be a regular file");
+ // Rows not yet checkpointed live in the WAL, so it counts toward the input cap.
+ let mut wal = input.as_os_str().to_owned();
+ wal.push("-wal");
+ let wal_len = fs::metadata(&wal).map(|m| m.len()).unwrap_or(0);
ensure!(
- meta.len() <= MAX_INPUT_BYTES,
+ meta.len() + wal_len <= MAX_INPUT_BYTES,
"claude-mem database exceeds 512 MiB; export a smaller copy"
);
let source_file = input
@@ -315,9 +325,8 @@ pub(super) fn import(input: &Path, output: &Path) -> Result<ImportSummary> {
.context("cannot open claude-mem database read-only")?;
conn.execute_batch("PRAGMA query_only = 1; BEGIN")
.context("cannot read claude-mem database")?;
- let tables = columns(&conn, "observations").map_err(|_| {
- anyhow::anyhow!("input is not a readable SQLite database; pass ~/.claude-mem/claude-mem.db")
- })?;
+ let tables = columns(&conn, "observations")
+ .context("cannot read the input as a SQLite database; pass ~/.claude-mem/claude-mem.db")?;
ensure!(
!tables.is_empty(),
"not a claude-mem database: no observations table"
@@ -459,6 +468,44 @@ mod tests {
drop(worker);
}
+ #[test]
+ fn claude_mem_other_platform_sessions_keep_their_id_and_prompts_export_once() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = store(dir.path());
+ // sdk_sessions repeats a content_session_id once per platform, and a
+ // non-Claude platform's session ID is not a UUID.
+ Connection::open(&input)
+ .unwrap()
+ .execute_batch(
+ "INSERT INTO sdk_sessions VALUES (2, '550e8400-e29b-41d4-a716-446655440000',
+ 'mem-0002', 'watcher', 'codex', NULL, '2026-10-25T00:00:00.000Z',
+ 1792886400000, 'completed', NULL, NULL);
+ INSERT INTO sdk_sessions VALUES (3, 'rollout-abc', 'mem-0003', 'watcher',
+ 'codex', NULL, '2026-10-25T00:00:00.000Z', 1792886400000, 'completed',
+ NULL, NULL);
+ UPDATE observations SET memory_session_id = 'mem-0003';",
+ )
+ .unwrap();
+ let output = dir.path().join("safe.jsonl");
+ let summary = import_claude_mem(&input, &output).unwrap();
+ assert_eq!((summary.records, summary.source_lines), (3, 3));
+ let rows = rows(&output);
+ let ids: Vec<_> = rows.iter().map(|r| r["id"].as_str().unwrap()).collect();
+ assert_eq!(
+ ids,
+ [
+ "claude-mem/user/1",
+ "claude-mem/observation/1",
+ "claude-mem/summary/1"
+ ]
+ );
+ assert_eq!(
+ rows[0]["session_id"],
+ "550e8400-e29b-41d4-a716-446655440000"
+ );
+ assert_eq!(rows[1]["session_id"], "rollout-abc");
+ }
+
#[test]
fn claude_mem_older_schema_without_migrated_columns_still_exports() {
let dir = tempfile::tempdir().unwrap();
@@ -492,13 +539,22 @@ mod tests {
.unwrap()
.execute_batch("CREATE TABLE t (x)")
.unwrap();
+ let pre_rename = dir.path().join("old.db");
+ Connection::open(&pre_rename)
+ .unwrap()
+ .execute_batch(
+ "CREATE TABLE observations (id INTEGER PRIMARY KEY, sdk_session_id TEXT,
+ project TEXT, text TEXT, type TEXT, created_at_epoch INTEGER);",
+ )
+ .unwrap();
let malformed = store(dir.path());
Connection::open(&malformed)
.unwrap()
.execute_batch("UPDATE observations SET facts = 'not json'")
.unwrap();
for (input, reason) in [
- (&not_sqlite, "not a readable SQLite database"),
+ (&not_sqlite, "cannot read the input as a SQLite database"),
+ (&pre_rename, "predates claude-mem's session ID rename"),
(&other, "no observations table"),
(
&malformed,
diff --git a/tools/session-import/README.md b/tools/session-import/README.md
index 4d778a202..54dc51dba 100644
--- a/tools/session-import/README.md
+++ b/tools/session-import/README.md
@@ -238,7 +238,7 @@ python3 tools/session-import/smoke.py --binary engine/target/release/xerj --form
[claude-mem](https://github.com/thedotmack/claude-mem) is a Claude Code plugin
that has a model condense each session into observations and summaries, kept in
one SQLite file. Pass that file explicitly; the command does not look for it,
-start or stop the claude-mem worker, or write to the store:
+start or stop the claude-mem worker, or change the store's contents:
```sh
set -eu
@@ -250,10 +250,14 @@ xerj sessions import --format claude-mem \
The store is opened read-only, not immutable, inside one read transaction, so
rows a running worker still holds in `claude-mem.db-wal` are included and all
-three tables come from the same snapshot. SQLite needs the store's `-shm` file
-or a writable folder to read a WAL database; on a read-only copy without either,
-copy the `.db`, `-wal` and `-shm` files together. A file with no `observations`
-table refuses as not a claude-mem store.
+three tables come from the same snapshot. SQLite may create the store's
+`-wal` and `-shm` side files to read it, and needs the `-shm` file or a
+writable folder to read a WAL database; on a read-only copy without either,
+copy the `.db`, `-wal` and `-shm` files together. The 512 MiB input cap counts
+the `.db` and `-wal` files together. A file with no `observations` table
+refuses as not a claude-mem store. A store from before claude-mem's session ID
+rename (its `sdk_session_id` columns) refuses with a request to start
+claude-mem once, which migrates it.
Three tables export, each ordered by `created_at_epoch` then row id:
@@ -275,7 +279,7 @@ format-specific meanings:
| Field | claude-mem meaning |
|---|---|
-| `session_format`, `session_id` | `claude-mem`; the Claude Code session UUID (`content_session_id`), joined through `sdk_sessions` for observations and summaries. A non-UUID value is kept as redacted text; no session row gives null |
+| `session_format`, `session_id` | `claude-mem`; the session ID claude-mem recorded for that row's platform (`content_session_id`, a UUID for Claude Code), joined through `sdk_sessions` on `memory_session_id` for observations and summaries. A non-UUID value is kept as redacted text; no session row gives null |
| `id`, `message_id` | `claude-mem/<role>/<row id>` and `<role>/<row id>`: stable for a row across exports of the same store |
| `parent_id` | null |
| `files` | `files_modified` (summaries: `files_edited`) then `files_read`, deduplicated, as claude-mem recorded them |
@@ -299,6 +303,7 @@ Reference retrieved before implementation: claude-mem, Apache-2.0, pinned at
- [`SessionStore.ts:1273-1325`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L1273-L1325): base `sdk_sessions`, `observations` and `session_summaries` tables and their session keys.
- [`SessionStore.ts:1697-1712`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L1697-L1712): `user_prompts`, keyed by `content_session_id` and `session_db_id`, with no project column.
+- [`SessionStore.ts:1812-1821`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L1812-L1821): migration 17, which renamed `sdk_session_id` to `memory_session_id` and `claude_session_id` to `content_session_id`.
- [`SessionStore.ts:4166-4180`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L4166-L4180): observation fields, with `facts`, `concepts` and `files_*` stored as `JSON.stringify` arrays.
These are format evidence, not copied implementation. The fixture