Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(cli): prepare Claude Code session text for autoindex (#2406)

Jarvisc62be98de6bd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e2374ea3b..2427cf3df 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- `xerj sessions import --format claude-code --input FILE --output NEW.jsonl`
+ prepares an explicit Claude Code JSONL transcript for ordinary `autoindex`.
+ It emits text-bearing user/assistant records through the existing native
+ PII plugin before writing, preserving UUIDs, parent links, source filename,
+ line/byte offset and explicitly referenced file paths. Tool bodies,
+ thinking and metadata records are omitted. The export is an offline,
+ new-file snapshot: malformed input, unsupported turn shapes, repeated UUIDs
+ or a changed source refuse publication; existing output is never replaced.
+ This covers one preparation format for #1793, not automatic session
+ discovery, incremental synchronization or universal secret removal. See
+ [the import guide](tools/session-import/README.md).
+
- With `[cluster] enabled = true`, `POST /v1/indices/{name}/docs` (one
document, or a JSON array with one document per element),
`DELETE /v1/indices/{name}/docs/{id}`, and
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index da00ba696..ff36d778d 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7459,8 +7459,10 @@ dependencies = [
"tree-sitter-swift",
"tree-sitter-typescript",
"tree-sitter-zig",
+ "uuid",
"walkdir",
"xerj-common",
+ "xerj-wasm",
"xxhash-rust",
"zip 9.0.5",
]
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index d4353452e..72c409f26 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -10,6 +10,9 @@ description = "xerj autoindex — point it at any folder and make the contents A
# The machine-resource policy (cores, memory safe zone, thread priority) is
# shared with the server: one policy, one place (#240).
xerj-common = { path = "../xerj-common" }
+# Session exports use the same native PII redactor as ingest pipelines.
+xerj-wasm = { path = "../xerj-wasm" }
+uuid.workspace = true
anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index b9775996d..367943c25 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -49,6 +49,7 @@ mod reconcile_plan;
pub mod remove;
pub mod resources;
pub mod search;
+pub mod sessions;
pub mod sniff;
pub mod source;
pub mod staleness;
diff --git a/engine/crates/xerj-autoindex/src/sessions.rs b/engine/crates/xerj-autoindex/src/sessions.rs
new file mode 100644
index 000000000..8767a7963
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/sessions.rs
@@ -0,0 +1,697 @@
+//! Explicit, offline session export before ordinary autoindex ingestion.
+use anyhow::{bail, ensure, Context, Result};
+use serde::Serialize;
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::collections::HashSet;
+use std::fs::{self, File, Metadata};
+use std::io::{BufRead, BufReader, Read, Write};
+use std::path::{Path, PathBuf};
+use xerj_wasm::{builtins::PiiRedactionPlugin, ProcessAction, TransformPlugin};
+
+const MAX_INPUT_BYTES: u64 = 512 * 1024 * 1024;
+const MAX_LINE_BYTES: u64 = 1024 * 1024;
+const MAX_TEXT_BYTES: usize = 256 * 1024;
+const MAX_RECORDS: usize = 100_000;
+
+pub const HELP: &str = "xerj sessions import — offline session preparation for autoindex
+
+USAGE: xerj sessions import --format claude-code --input FILE --output NEW.jsonl
+
+Read one explicit Claude Code JSONL transcript; write user/assistant text
+records through the existing native PII redactor before any output is stored.
+Preserve session/message UUIDs, parent UUIDs, source filename/line/byte offset,
+branch metadata and file paths explicitly named by Read/Write/Edit/MultiEdit.
+Skip tool results, tool inputs except file paths, thinking and metadata.
+This exports all text-bearing branches in file order, not an active conversation
+chain. Unsupported/malformed turns or repeated message UUIDs refuse the export.
+
+The output is a new, private JSONL snapshot. Existing output is never replaced;
+parse failures publish nothing. No node is started or contacted. Review the
+export, then run: xerj autoindex <export-folder> --prefix <your-prefix>
+The redactor handles email/IP/card/SSN/phone patterns, not every secret or name.
+Input <=512 MiB; line <=1 MiB; turn text <=256 KiB; <=100000 message UUIDs.
+Exit 0: exported, JSON summary on stdout; 2: usage; 1: import/publication error.
+";
+
+#[derive(Debug, Serialize)]
+pub struct ImportSummary {
+ pub schema_version: u32,
+ pub records: u64,
+ pub source_lines: u64,
+ pub skipped_records: u64,
+ pub skipped_blocks: u64,
+ pub source_sha256: String,
+ pub output_sha256: String,
+}
+
+fn uuid_field(entry: &Value, field: &str, line: u64) -> Result<String> {
+ let text = entry[field]
+ .as_str()
+ .filter(|s| s.len() == 36)
+ .with_context(|| format!("line {line}: expected UUID in {field}"))?;
+ uuid::Uuid::parse_str(text)
+ .map(|id| id.to_string())
+ .map_err(|_| anyhow::anyhow!("line {line}: invalid UUID in {field}"))
+}
+
+fn optional_text(entry: &Value, field: &str, line: u64) -> Result<Value> {
+ match entry.get(field) {
+ None | Some(Value::Null) => Ok(Value::Null),
+ Some(Value::String(s)) => Ok(json!(s)),
+ _ => bail!("line {line}: expected text in {field}"),
+ }
+}
+
+fn flag(entry: &Value, field: &str, line: u64) -> Result<bool> {
+ match entry.get(field) {
+ None | Some(Value::Null) => Ok(false),
+ Some(Value::Bool(value)) => Ok(*value),
+ _ => bail!("line {line}: expected boolean in {field}"),
+ }
+}
+
+fn turn_text(content: &Value, line: u64, skipped: &mut u64) -> Result<(String, Vec<String>)> {
+ if let Some(text) = content.as_str() {
+ ensure!(
+ text.len() <= MAX_TEXT_BYTES,
+ "line {line}: turn text exceeds 256 KiB"
+ );
+ return Ok((text.to_string(), Vec::new()));
+ }
+ let blocks = content
+ .as_array()
+ .with_context(|| format!("line {line}: unsupported message content"))?;
+ let mut text = String::new();
+ let mut files = Vec::new();
+ for block in blocks {
+ let kind = block["type"]
+ .as_str()
+ .with_context(|| format!("line {line}: missing content block type"))?;
+ if kind == "text" {
+ let part = block["text"]
+ .as_str()
+ .with_context(|| format!("line {line}: malformed text block"))?;
+ let separator_bytes = usize::from(!text.is_empty());
+ ensure!(
+ text.len() + part.len() + separator_bytes <= MAX_TEXT_BYTES,
+ "line {line}: turn text exceeds 256 KiB"
+ );
+ if !text.is_empty() {
+ text.push('\n');
+ }
+ text.push_str(part);
+ } else {
+ ensure!(
+ matches!(
+ kind,
+ "tool_use"
+ | "server_tool_use"
+ | "tool_result"
+ | "thinking"
+ | "redacted_thinking"
+ | "image"
+ | "document"
+ | "container_upload"
+ ) || kind.ends_with("_tool_result"),
+ "line {line}: unsupported content block type"
+ );
+ *skipped += 1;
+ if kind == "tool_use"
+ && matches!(
+ block["name"].as_str(),
+ Some("Read" | "Write" | "Edit" | "MultiEdit")
+ )
+ {
+ if let Some(path) = block.pointer("/input/file_path") {
+ let path = path
+ .as_str()
+ .with_context(|| format!("line {line}: malformed tool file path"))?;
+ if !files.iter().any(|f| f == path) {
+ files.push(path.to_string());
+ }
+ }
+ }
+ }
+ }
+ Ok((text, files))
+}
+
+fn unchanged(file: &File, before: &Metadata, input: &Path, read: u64) -> Result<()> {
+ let after = file.metadata()?;
+ let named = fs::metadata(input)?;
+ ensure!(
+ before.len() == read
+ && before.len() == after.len()
+ && before.len() == named.len()
+ && before.modified()? == after.modified()?
+ && before.modified()? == named.modified()?,
+ "session input changed during import; use a stopped or archived transcript"
+ );
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::MetadataExt;
+ ensure!(
+ before.dev() == named.dev() && before.ino() == named.ino(),
+ "session input was replaced during import; use an archived transcript"
+ );
+ }
+ Ok(())
+}
+
+fn publish(file: tempfile::NamedTempFile, output: &Path) -> Result<()> {
+ file.persist_noclobber(output)
+ .map_err(|_| anyhow::anyhow!("cannot publish session export: output exists or is not writable; choose a new output path"))?;
+ Ok(())
+}
+
+pub fn import_claude_code(input: &Path, output: &Path) -> Result<ImportSummary> {
+ match fs::symlink_metadata(output) {
+ Ok(_) => bail!("session output already exists; choose a new output path"),
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+ Err(e) => return Err(e).context("cannot inspect session output"),
+ }
+ let mut options = fs::OpenOptions::new();
+ options.read(true);
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::OpenOptionsExt;
+ // Opening a FIFO must not block before we can reject non-files.
+ options.custom_flags(libc::O_NONBLOCK);
+ }
+ let file = options.open(input).context("cannot open session input")?;
+ let before = file.metadata()?;
+ ensure!(before.is_file(), "session input must be a regular file");
+ ensure!(
+ before.len() <= MAX_INPUT_BYTES,
+ "session input exceeds 512 MiB; split or archive a smaller transcript"
+ );
+ let source_file = input
+ .file_name()
+ .and_then(|s| s.to_str())
+ .context("session input filename must be UTF-8")?;
+ let parent = output
+ .parent()
+ .filter(|p| !p.as_os_str().is_empty())
+ .unwrap_or(Path::new("."));
+ let mut staging = tempfile::Builder::new()
+ .prefix(".xerj-session-")
+ .suffix(".tmp")
+ .tempfile_in(parent)
+ .context("cannot create private export staging file; output parent must exist")?;
+ let redactor = PiiRedactionPlugin::new(Vec::new());
+ let mut summary = ImportSummary {
+ schema_version: 1,
+ records: 0,
+ source_lines: 0,
+ skipped_records: 0,
+ skipped_blocks: 0,
+ source_sha256: String::new(),
+ output_sha256: String::new(),
+ };
+ let mut source_hash = Sha256::new();
+ let mut output_hash = Sha256::new();
+ let mut reader = BufReader::new((&file).take(before.len() + 1));
+ let mut bytes = Vec::new();
+ let mut read = 0;
+ let mut written = 0u64;
+ let mut session: Option<String> = None;
+ let mut seen = HashSet::new();
+ loop {
+ bytes.clear();
+ let count = (&mut reader)
+ .take(MAX_LINE_BYTES + 1)
+ .read_until(b'\n', &mut bytes)?;
+ if count == 0 {
+ break;
+ }
+ summary.source_lines += 1;
+ let line = summary.source_lines;
+ let offset = read;
+ read += count as u64;
+ ensure!(
+ count as u64 <= MAX_LINE_BYTES,
+ "line {line}: session line exceeds 1 MiB"
+ );
+ source_hash.update(&bytes);
+ if bytes.iter().all(u8::is_ascii_whitespace) {
+ continue;
+ }
+ let entry: Value = serde_json::from_slice(&bytes).map_err(|_| {
+ anyhow::anyhow!("line {line}: invalid JSON; no session export published")
+ })?;
+ let kind = entry["type"]
+ .as_str()
+ .with_context(|| format!("line {line}: missing session record type"))?;
+ if !matches!(kind, "user" | "assistant") {
+ summary.skipped_records += 1;
+ continue;
+ }
+ let sid = uuid_field(&entry, "sessionId", line)?;
+ if let Some(expected) = &session {
+ ensure!(
+ expected == &sid,
+ "line {line}: mixed session IDs are unsupported"
+ );
+ } else {
+ session = Some(sid.clone());
+ }
+ let mid = uuid_field(&entry, "uuid", line)?;
+ ensure!(
+ seen.len() < MAX_RECORDS,
+ "session exceeds 100000 message UUIDs"
+ );
+ ensure!(
+ seen.insert(mid.clone()),
+ "line {line}: repeated message UUID; export a transcript without revisions"
+ );
+ if flag(&entry, "isMeta", line)? {
+ summary.skipped_records += 1;
+ continue;
+ }
+ let message = entry["message"]
+ .as_object()
+ .with_context(|| format!("line {line}: missing message object"))?;
+ ensure!(
+ message.get("role").is_none_or(|r| r.as_str() == Some(kind)),
+ "line {line}: message role disagrees with record type"
+ );
+ let (body, files) = turn_text(
+ message.get("content").unwrap_or(&Value::Null),
+ line,
+ &mut summary.skipped_blocks,
+ )?;
+ if body.trim().is_empty() {
+ summary.skipped_records += 1;
+ continue;
+ }
+ let parent_id = if entry["parentUuid"].is_null() {
+ Value::Null
+ } else {
+ json!(uuid_field(&entry, "parentUuid", line)?)
+ };
+ let timestamp = optional_text(&entry, "timestamp", line)?;
+ if let Some(text) = timestamp.as_str() {
+ ensure!(
+ chrono::DateTime::parse_from_rfc3339(text).is_ok(),
+ "line {line}: invalid timestamp"
+ );
+ }
+ let mut record = json!({"body":body, "files":files, "source_file":source_file,
+ "cwd":optional_text(&entry,"cwd",line)?, "git_branch":optional_text(&entry,"gitBranch",line)?});
+ ensure!(
+ redactor.process(&mut record) == ProcessAction::Pass,
+ "PII redaction refused session record"
+ );
+ // Only validated opaque IDs/enums and provenance numbers bypass the
+ // text redactor: a phone regex must not corrupt UUID relationships.
+ record["id"] = json!(format!("{sid}/{mid}"));
+ record["session_schema_version"] = json!(1);
+ record["session_format"] = json!("claude-code");
+ record["session_id"] = json!(sid);
+ record["message_id"] = json!(mid);
+ record["parent_id"] = parent_id;
+ record["role"] = json!(kind);
+ record["timestamp"] = timestamp;
+ record["source_line"] = json!(line);
+ record["source_byte_offset"] = json!(offset);
+ record["is_sidechain"] = json!(flag(&entry, "isSidechain", line)?);
+ record["is_summary"] = json!(flag(&entry, "isCompactSummary", line)?);
+ let title = record["body"]
+ .as_str()
+ .unwrap()
+ .lines()
+ .next()
+ .unwrap_or("")
+ .chars()
+ .take(120)
+ .collect::<String>();
+ record["title"] = json!(format!("{kind}: {title}"));
+ let mut encoded = serde_json::to_vec(&record)?;
+ encoded.push(b'\n');
+ written += encoded.len() as u64;
+ ensure!(
+ written <= MAX_INPUT_BYTES,
+ "session export exceeds 512 MiB; use a smaller transcript"
+ );
+ staging.write_all(&encoded)?;
+ output_hash.update(&encoded);
+ summary.records += 1;
+ }
+ ensure!(
+ summary.records > 0,
+ "no supported text-bearing user/assistant turns; no export published"
+ );
+ unchanged(&file, &before, input, read)?;
+ summary.source_sha256 = format!("{:x}", source_hash.finalize());
+ summary.output_sha256 = format!("{:x}", output_hash.finalize());
+ staging.as_file_mut().sync_all()?;
+ publish(staging, output)?;
+ Ok(summary)
+}
+
+pub fn run_cli(args: &[String]) -> i32 {
+ if args.is_empty() || args.iter().any(|a| matches!(a.as_str(), "--help" | "-h")) {
+ print!("{HELP}");
+ return 0;
+ }
+ let parsed = (|| -> Result<(PathBuf, PathBuf)> {
+ ensure!(args[0] == "import", "expected sessions import");
+ let mut input = None;
+ let mut output = None;
+ let mut format = None;
+ let mut it = args[1..].iter();
+ while let Some(arg) = it.next() {
+ let slot = match arg.as_str() {
+ "--input" => &mut input,
+ "--output" => &mut output,
+ "--format" => &mut format,
+ _ => bail!("unknown session import option; see xerj sessions --help"),
+ };
+ ensure!(slot.is_none(), "duplicate session import option");
+ let value = it
+ .next()
+ .filter(|v| !v.is_empty() && !v.starts_with("--"))
+ .context("session import option requires a value")?;
+ *slot = Some(value.clone());
+ }
+ ensure!(
+ format.as_deref() == Some("claude-code"),
+ "--format claude-code is required"
+ );
+ Ok((
+ PathBuf::from(input.context("--input is required")?),
+ PathBuf::from(output.context("--output is required")?),
+ ))
+ })();
+ let (input, output) = match parsed {
+ Ok(paths) => paths,
+ Err(error) => {
+ eprintln!("error: {error}");
+ return 2;
+ }
+ };
+ match import_claude_code(&input, &output) {
+ Ok(summary) => {
+ let mut stdout = std::io::stdout().lock();
+ if serde_json::to_writer(&mut stdout, &summary).is_err() || writeln!(stdout).is_err() {
+ eprintln!("error: export was published, but its summary could not be written");
+ return 1;
+ }
+ 0
+ }
+ Err(error) => {
+ eprintln!("error: {error:#}");
+ 1
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::{json, Value};
+
+ const SESSION: &str = "550e8400-e29b-41d4-a716-446655440000";
+ const USER: &str = "550e8400-e29b-41d4-a716-446655440001";
+ const ASSISTANT: &str = "550e8400-e29b-41d4-a716-446655440002";
+
+ fn message(role: &str, uuid: &str, content: Value) -> Value {
+ json!({"type":role, "uuid":uuid, "sessionId":SESSION,
+ "timestamp":"2026-10-25T00:00:00Z",
+ "message":{"role":role,"content":content}})
+ }
+
+ fn write_input(path: &Path, entries: &[Value]) {
+ let mut text = entries
+ .iter()
+ .map(Value::to_string)
+ .collect::<Vec<_>>()
+ .join("\n");
+ text.push('\n');
+ std::fs::write(path, text).unwrap();
+ }
+
+ #[test]
+ fn sessions_decision_text_is_redacted_and_points_to_the_original_turn() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("conversation.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ let mut decision = message(
+ "assistant",
+ ASSISTANT,
+ json!([
+ {"type":"thinking","thinking":"NEVER_EXPORT_THOUGHTS"},
+ {"type":"text","text":"We dropped polling because file events preserve recovery. Contact [email protected]."},
+ {"type":"tool_use","name":"Edit","input":{"file_path":"src/watch.rs","old_string":"NEVER_EXPORT_INPUT"}}
+ ]),
+ );
+ decision["parentUuid"] = json!(USER);
+ write_input(
+ &input,
+ &[
+ json!({"type":"file-history-snapshot","payload":"NEVER_EXPORT_METADATA"}),
+ message("user", USER, json!("Why did we drop polling?")),
+ decision,
+ ],
+ );
+ let summary = import_claude_code(&input, &output).unwrap();
+ assert_eq!(summary.records, 2);
+ let text = std::fs::read_to_string(&output).unwrap();
+ assert!(text.contains("[REDACTED_EMAIL]"));
+ for secret in [
+ "NEVER_EXPORT_THOUGHTS",
+ "NEVER_EXPORT_INPUT",
+ "NEVER_EXPORT_METADATA",
+ ] {
+ assert!(!text.contains(secret), "export leaked {secret}");
+ }
+ let rows: Vec<Value> = text
+ .lines()
+ .map(|l| serde_json::from_str(l).unwrap())
+ .collect();
+ assert_eq!(rows[1]["source_line"], 3);
+ assert_eq!(rows[1]["message_id"], ASSISTANT);
+ assert_eq!(rows[1]["parent_id"], USER);
+ assert_eq!(rows[1]["files"], json!(["src/watch.rs"]));
+ assert_eq!(
+ rows[0].as_object().unwrap().keys().collect::<Vec<_>>(),
+ rows[1].as_object().unwrap().keys().collect::<Vec<_>>()
+ );
+ }
+
+ #[test]
+ fn sessions_a_malformed_later_line_publishes_nothing() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ write_input(&input, &[message("user", USER, json!("valid text"))]);
+ use std::io::Write;
+ std::fs::OpenOptions::new()
+ .append(true)
+ .open(&input)
+ .unwrap()
+ .write_all(b"{\"secret\":\"NEVER_ECHO_THIS\"")
+ .unwrap();
+ let error = import_claude_code(&input, &output).unwrap_err().to_string();
+ assert!(error.contains("line 2"), "{error}");
+ assert!(!error.contains("NEVER_ECHO_THIS"));
+ assert!(!output.exists());
+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);
+ }
+
+ #[test]
+ fn sessions_existing_output_is_preserved() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ write_input(&input, &[message("user", USER, json!("hello"))]);
+ std::fs::write(&output, "existing").unwrap();
+ assert!(import_claude_code(&input, &output).is_err());
+ assert_eq!(std::fs::read_to_string(&output).unwrap(), "existing");
+ }
+
+ #[test]
+ fn sessions_redacts_the_five_native_pii_types_and_metadata() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("[email protected]");
+ let output = dir.path().join("safe.jsonl");
+ let mut entry = message("user", USER, json!("Email: [email protected]. IP: 192.0.2.55. Card: 4111-1111-1111-1111. SSN: 123-45-6789. Phone: 202-555-0147."));
+ entry["cwd"] = json!("/home/[email protected]/work");
+ entry["gitBranch"] = json!("contact/[email protected]");
+ write_input(&input, &[entry]);
+ let summary = import_claude_code(&input, &output).unwrap();
+ let bytes = std::fs::read(&output).unwrap();
+ let text = std::str::from_utf8(&bytes).unwrap();
+ for marker in ["EMAIL", "IP", "CC", "SSN", "PHONE"] {
+ assert!(text.contains(&format!("[REDACTED_{marker}]")), "{text}");
+ }
+ assert!(!text.contains("[email protected]"));
+ let row: Value = serde_json::from_slice(&bytes).unwrap();
+ assert_eq!(row["session_id"], SESSION);
+ assert_eq!(row["message_id"], USER);
+ assert_eq!(
+ summary.source_sha256,
+ format!("{:x}", Sha256::digest(std::fs::read(input).unwrap()))
+ );
+ assert_eq!(
+ summary.output_sha256,
+ format!("{:x}", Sha256::digest(bytes))
+ );
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ assert_eq!(
+ std::fs::metadata(output).unwrap().permissions().mode() & 0o777,
+ 0o600
+ );
+ }
+ }
+
+ #[test]
+ fn sessions_duplicate_mixed_or_unknown_turns_fail_without_partial_output() {
+ let first = message("user", USER, json!("first"));
+ let second = message("assistant", ASSISTANT, json!("second"));
+ let mut mixed = second.clone();
+ mixed["sessionId"] = json!("550e8400-e29b-41d4-a716-446655440099");
+ let mut unknown = second.clone();
+ unknown["message"]["content"] = json!([{"type":"future_text","text":"unsupported"}]);
+ let mut bad_role = second.clone();
+ bad_role["message"]["role"] = json!("user");
+ for (entry, needle) in [
+ (first.clone(), "repeated message UUID"),
+ (mixed, "mixed session IDs"),
+ (unknown, "unsupported content block"),
+ (bad_role, "role disagrees"),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ write_input(&input, &[first.clone(), entry]);
+ let error = import_claude_code(&input, &output).unwrap_err().to_string();
+ assert!(error.contains(needle), "{error}");
+ assert!(!output.exists());
+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);
+ // Recovery uses the same never-published output after replacing
+ // the synthetic malformed input, not weakening a parser guard.
+ write_input(&input, &[first.clone(), second.clone()]);
+ assert_eq!(import_claude_code(&input, &output).unwrap().records, 2);
+ }
+ }
+
+ #[test]
+ fn sessions_meta_and_tool_only_messages_do_not_become_searchable_text() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ let mut meta = message("assistant", ASSISTANT, json!("hidden metadata text"));
+ meta["isMeta"] = json!(true);
+ write_input(
+ &input,
+ &[
+ message(
+ "user",
+ USER,
+ json!([{"type":"tool_result","content":"raw result"}]),
+ ),
+ meta,
+ ],
+ );
+ assert!(import_claude_code(&input, &output)
+ .unwrap_err()
+ .to_string()
+ .contains("no supported text-bearing"));
+ assert!(!output.exists());
+ }
+
+ #[test]
+ fn sessions_preserves_branch_flags_and_ids_without_choosing_a_live_leaf() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ let mut entry = message("assistant", ASSISTANT, json!("discarded alternative"));
+ entry["isSidechain"] = json!(true);
+ entry["isCompactSummary"] = json!(true);
+ entry["parentUuid"] = json!(USER);
+ write_input(&input, &[entry]);
+ import_claude_code(&input, &output).unwrap();
+ let row: Value = serde_json::from_slice(&std::fs::read(output).unwrap()).unwrap();
+ assert_eq!(row["is_sidechain"], true);
+ assert_eq!(row["is_summary"], true);
+ assert_eq!(row["parent_id"], USER);
+ }
+
+ #[test]
+ fn sessions_input_change_or_output_creation_cannot_publish_over_existing_state() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ std::fs::write(&input, "before").unwrap();
+ let opened = File::open(&input).unwrap();
+ let before = opened.metadata().unwrap();
+ std::fs::write(&input, "a different length").unwrap();
+ assert!(unchanged(&opened, &before, &input, before.len())
+ .unwrap_err()
+ .to_string()
+ .contains("changed during import"));
+ let output = dir.path().join("safe.jsonl");
+ let mut staging = tempfile::NamedTempFile::new_in(dir.path()).unwrap();
+ staging.write_all(b"safe new text").unwrap();
+ // Simulate another writer winning after preflight, before publication.
+ std::fs::write(&output, "winner").unwrap();
+ assert!(publish(staging, &output).is_err());
+ assert_eq!(std::fs::read_to_string(output).unwrap(), "winner");
+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2);
+ }
+
+ #[test]
+ fn sessions_limits_refuse_instead_of_truncating_or_falling_back_to_raw_text() {
+ for payload in [
+ vec![b'x'; MAX_LINE_BYTES as usize + 1],
+ message("user", USER, json!("x".repeat(MAX_TEXT_BYTES + 1)))
+ .to_string()
+ .into_bytes(),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ std::fs::write(&input, payload).unwrap();
+ let error = import_claude_code(&input, &output).unwrap_err().to_string();
+ assert!(error.contains("exceeds"), "{error}");
+ assert!(!output.exists());
+ }
+ }
+
+ #[test]
+ fn sessions_text_limit_is_inclusive_for_strings_and_block_arrays() {
+ for content in [
+ json!("x".repeat(MAX_TEXT_BYTES)),
+ json!([
+ {"type":"text","text":"x".repeat(MAX_TEXT_BYTES)}
+ ]),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.jsonl");
+ let output = dir.path().join("safe.jsonl");
+ write_input(&input, &[message("user", USER, content)]);
+ assert_eq!(import_claude_code(&input, &output).unwrap().records, 1);
+ let row: Value = serde_json::from_slice(&std::fs::read(output).unwrap()).unwrap();
+ assert_eq!(row["body"].as_str().unwrap().len(), MAX_TEXT_BYTES);
+ }
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn sessions_named_pipe_is_rejected_without_waiting_for_a_writer() {
+ use std::os::unix::ffi::OsStrExt;
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("pipe");
+ let output = dir.path().join("safe.jsonl");
+ let name = std::ffi::CString::new(input.as_os_str().as_bytes()).unwrap();
+ // SAFETY: the CString is a valid NUL-terminated temporary path.
+ assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o600) }, 0);
+ let error = import_claude_code(&input, &output).unwrap_err().to_string();
+ assert!(error.contains("regular file"), "{error}");
+ assert!(!output.exists());
+ }
+}
diff --git a/engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonl b/engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonl
new file mode 100644
index 000000000..05a928653
--- /dev/null
+++ b/engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonl
@@ -0,0 +1,4 @@
+{"type":"file-history-snapshot","payload":"DO_NOT_EXPORT_METADATA"}
+{"type":"user","sessionId":"550e8400-e29b-41d4-a716-446655440000","uuid":"550e8400-e29b-41d4-a716-446655440001","timestamp":"2026-10-25T00:00:00Z","message":{"role":"user","content":"Why did we drop polling? Contact [email protected] or 202-555-0147."}}
+{"type":"assistant","sessionId":"550e8400-e29b-41d4-a716-446655440000","uuid":"550e8400-e29b-41d4-a716-446655440002","parentUuid":"550e8400-e29b-41d4-a716-446655440001","timestamp":"2026-10-25T00:00:01Z","gitBranch":"feature/watch","message":{"role":"assistant","content":[{"type":"thinking","thinking":"DO_NOT_EXPORT_THINKING"},{"type":"text","text":"We dropped polling because file events avoid repeated scans while the durable journal preserves recovery."},{"type":"tool_use","name":"Edit","input":{"file_path":"src/watch.rs","new_string":"DO_NOT_EXPORT_TOOL_INPUT"}}]}}
+{"type":"user","sessionId":"550e8400-e29b-41d4-a716-446655440000","uuid":"550e8400-e29b-41d4-a716-446655440003","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"synthetic-tool","content":"DO_NOT_EXPORT_TOOL_OUTPUT"}]}}
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 86c49621b..8d83fec68 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -173,6 +173,7 @@ fn is_top_level_command(arg: &str) -> bool {
"index"
| "search"
| "batch-search"
+ | "sessions"
| "def"
| "init"
| "gain"
@@ -451,6 +452,7 @@ fn help_text(feedback: bool) -> String {
\n\
SUBCOMMANDS:\n\
\x20\x20\x20\x20xerj batch-search --input FILE ordered batch retrieval, JSON in/out (see xerj batch-search --help)\n\
+ \x20\x20\x20\x20xerj sessions import export selected session turns with PII redaction (see xerj sessions --help)\n\
\x20\x20\x20\x20xerj index <opts> direct NDJSON → engine ingest (see xerj index --help)\n\
\x20\x20\x20\x20xerj autoindex <folder> [opts] zero-config folder discovery + indexing (see xerj autoindex --help)\n\
\x20\x20\x20\x20xerj autoindex map [folder] data map of indexed data, or a read-only folder preview\n\
@@ -2936,6 +2938,10 @@ fn main() -> Result<()> {
let rt = build_runtime(2)?;
std::process::exit(rt.block_on(xerj_mcp::run_batch_cli(&args)));
}
+ if matches!(xerj_common::argv::subcommand().as_deref(), Some("sessions")) {
+ let args = xerj_common::argv::args_after_subcommand();
+ std::process::exit(xerj_autoindex::sessions::run_cli(&args));
+ }
// Cores from the resource policy, so `XERJ_NUM_CPUS` and a container CPU
// quota reach the async runtime the same way they reach every pool (#240).
let cores = xerj_common::resource::cores();
diff --git a/engine/crates/xerj-server/tests/sessions_cli.rs b/engine/crates/xerj-server/tests/sessions_cli.rs
new file mode 100644
index 000000000..e75bcb59a
--- /dev/null
+++ b/engine/crates/xerj-server/tests/sessions_cli.rs
@@ -0,0 +1,82 @@
+//! Offline CLI export: success, redaction and failure publication boundaries.
+use serde_json::Value;
+use std::path::Path;
+use std::process::{Command, Output};
+
+fn import(input: &Path, output: &Path) -> Output {
+ Command::new(env!("CARGO_BIN_EXE_xerj"))
+ .args(["sessions", "import", "--format", "claude-code", "--input"])
+ .arg(input)
+ .arg("--output")
+ .arg(output)
+ .env("XERJ_DISABLE_FEEDBACK", "true")
+ .output()
+ .unwrap()
+}
+
+#[test]
+fn sessions_cli_exports_only_redacted_conversation_and_refuses_overwrite() {
+ let work = tempfile::tempdir().unwrap();
+ let input = work.path().join("session.jsonl");
+ std::fs::write(
+ &input,
+ include_str!("../../xerj-autoindex/tests/fixtures/claude-session.jsonl"),
+ )
+ .unwrap();
+ let output = work.path().join("records.jsonl");
+ let result = import(&input, &output);
+ assert!(
+ result.status.success(),
+ "{}",
+ String::from_utf8_lossy(&result.stderr)
+ );
+ let report: Value = serde_json::from_slice(&result.stdout).unwrap();
+ assert_eq!(report["schema_version"], 1);
+ assert_eq!(report["records"], 2);
+ assert_eq!(report["source_lines"], 4);
+ assert_eq!(report["skipped_records"], 2);
+ assert_eq!(report["skipped_blocks"], 3);
+ let bytes = std::fs::read(&output).unwrap();
+ let text = std::str::from_utf8(&bytes).unwrap();
+ assert!(text.contains("[REDACTED_EMAIL]"));
+ assert!(!text.contains("[email protected]"));
+ assert!(!text.contains("DO_NOT_EXPORT"));
+ let denied = import(&input, &output);
+ assert_eq!(denied.status.code(), Some(1));
+ assert_eq!(std::fs::read(output).unwrap(), bytes);
+}
+
+#[test]
+fn sessions_cli_bad_format_is_usage_error_without_output() {
+ let work = tempfile::tempdir().unwrap();
+ let output = work.path().join("safe.jsonl");
+ let result = Command::new(env!("CARGO_BIN_EXE_xerj"))
+ .args([
+ "sessions", "import", "--format", "unknown", "--input", "unused", "--output",
+ ])
+ .arg(&output)
+ .output()
+ .unwrap();
+ assert_eq!(result.status.code(), Some(2));
+ assert!(!output.exists());
+}
+
+#[test]
+fn sessions_cli_malformed_tail_keeps_input_and_publishes_no_partial_export() {
+ let work = tempfile::tempdir().unwrap();
+ let input = work.path().join("session.jsonl");
+ let output = work.path().join("records.jsonl");
+ let mut bytes =
+ include_bytes!("../../xerj-autoindex/tests/fixtures/claude-session.jsonl").to_vec();
+ bytes.extend_from_slice(b"{\"private\":\"NEVER_ECHO_PRIVATE_PAYLOAD\"");
+ std::fs::write(&input, &bytes).unwrap();
+ let result = import(&input, &output);
+ assert_eq!(result.status.code(), Some(1));
+ assert!(result.stdout.is_empty());
+ let error = String::from_utf8_lossy(&result.stderr);
+ assert!(error.contains("invalid JSON"), "{error}");
+ assert!(!error.contains("NEVER_ECHO_PRIVATE_PAYLOAD"));
+ assert!(!output.exists());
+ assert_eq!(std::fs::read(input).unwrap(), bytes);
+ assert_eq!(std::fs::read_dir(work.path()).unwrap().count(), 1);
+}
diff --git a/tools/session-import/README.md b/tools/session-import/README.md
new file mode 100644
index 000000000..0049bc0aa
--- /dev/null
+++ b/tools/session-import/README.md
@@ -0,0 +1,148 @@
+# Import one Claude Code transcript for decision retrieval
+
+`xerj sessions import` is the explicit preparation step for a single Claude Code
+JSONL transcript. It runs locally, calls no model or node, and exports text
+turns through XERJ's existing native PII plugin. Review the export, then give its
+folder to ordinary `xerj autoindex`. This is the first scoped part of
+[#1793](https://github.com/xerj-org/xerj/issues/1793), not automatic transcript
+discovery, a multi-client adapter, a live conversation view or incremental sync.
+
+## Try the synthetic example
+
+From the repository root with a built binary and Python 3:
+
+```sh
+(cd engine && cargo build --locked --release -j 32 -p xerj-server)
+python3 tools/session-import/smoke.py --binary engine/target/release/xerj
+```
+
+The example uses only the committed synthetic transcript, prepares a private
+export, starts a private loopback node on unused ports, dry-runs autoindex,
+indexes and queries the decision, repeats indexing to check idempotence, then
+checks graceful restart. It removes its own temporary data at exit. It neither
+contacts nor removes any existing node/index. The node runs in default lexical
+mode; indexing explicitly uses `--no-semantic`. No semantic-quality claim is made.
+
+## Import your own archived transcript
+
+Choose a stopped/archived file explicitly; this command does not inspect your
+home directory or discover other sessions. Use a private destination directory
+whose parent you control. The output file must not exist, even if empty.
+
+```sh
+set -eu
+SESSION_EXPORT=$(mktemp -d)
+xerj sessions import --format claude-code \
+ --input /path/to/archived-session.jsonl \
+ --output "$SESSION_EXPORT/records.jsonl"
+```
+
+Success prints a JSON summary containing the schema version, exported-record
+count, physical source-line count, skipped-record count, skipped-block count,
+and SHA-256 of the source bytes consumed and the exact exported bytes. Blank
+lines count as source lines but not skipped records. Skipped blocks count the
+non-text blocks inspected in non-meta user/assistant messages. Skipped records
+count other top-level records, `isMeta` messages and messages with no text.
+
+The redactor handles the existing email, IP, credit-card, US SSN and phone
+patterns. It does not remove every credential, person name, repository path or
+private fact. Check the exported JSONL before indexing it. Raw tool payloads,
+tool results, thinking and arbitrary message metadata are excluded, including
+fields the importer does not recognize. The importer does not change the input.
+
+After review, against your running node:
+
+```sh
+set -eu
+xerj autoindex "$SESSION_EXPORT" --prefix session-history --dry-run
+# Check the estimate and prefix ownership before indexing.
+xerj autoindex "$SESSION_EXPORT" --prefix session-history
+xerj autoindex map --prefix session-history
+xerj search --prefix session-history "why did we drop polling"
+```
+
+Only put prepared exports in that folder. Ordinary `autoindex` still accepts
+raw JSONL; pointing it at the original transcript folder does **not** activate
+this preparation or redaction. Use a prefix you own; follow the normal
+autoindex estimate/approval contract for large exports.
+
+## Record and source contract
+
+One output line is one text-bearing `user` or `assistant` event. `message.content`
+may be a string or an array of `text`/recognized non-text blocks. Text blocks
+retain their order. Each output object has the same key set:
+
+| Fields | Meaning |
+|---|---|
+| `session_schema_version`, `session_format` | Export schema 1, `claude-code` |
+| `id`, `session_id`, `message_id` | Canonical validated UUIDs; ID is session/message UUID |
+| `parent_id` | Explicit `parentUuid`, or null; it may reference a skipped event |
+| `role`, `title`, `body` | Event role and redacted text; title is the first text line, capped at 120 characters |
+| `source_file`, `source_line`, `source_byte_offset` | Redacted original basename, 1-based physical line, 0-based byte offset |
+| `files` | Redacted `file_path` values explicitly requested by Read/Write/Edit/MultiEdit blocks in this event |
+| `cwd`, `git_branch`, `timestamp` | Explicit metadata or null; timestamp must be RFC 3339 |
+| `is_sidechain`, `is_summary` | Explicit branch/compaction flags, default false |
+
+File paths are references in a tool request, not proof that a change succeeded.
+No files are opened through these references, and no commit is inferred from
+prose, shell commands or branch names. File references in tool-only events are
+not exported in this text-only version. Autoindex citations still identify the
+prepared file/record; the `source_*` fields locate the original event, whose
+byte offset is meaningful only for that archived source digest. Redaction may
+change the original filename/path spelling. Query the stored record with
+`_source: true` to read all source metadata.
+
+The importer preserves all text-bearing branches in file order. It does not
+reconstruct the active conversation, merge forks or claim that a discarded
+alternative is current. Compaction summaries are marked, not treated as original
+turns. Non-text attachments are not OCRed, and tool results are not searched.
+
+## Refusals, reruns and limits
+
+Exit 0 is a complete export; 2 is invalid CLI usage; 1 is an import or
+publication/reporting error. If stdout fails after publication, stderr explicitly
+says the export was published: inspect and reuse that complete file rather than
+expecting a second import to overwrite it. Diagnostics name the line and schema problem, without
+echoing the source payload. Invalid JSON (including a truncated tail), UUIDs,
+roles, timestamps or flags, unsupported content blocks, mixed session IDs,
+repeated message UUIDs and a changed source refuse the whole export. There is
+no fallback to unredacted generic JSONL/text. Metadata-only input also refuses.
+
+Input and output are capped at 512 MiB each, source lines at 1 MiB including
+newline, collected turn text at 256 KiB and message UUIDs at 100,000. These are
+admission limits, not a measured peak-memory claim. Parsing is line by line.
+The temporary output contains only redacted, allowlisted records, is hidden
+and is removed on ordinary failure. It is synced before no-clobber publication;
+the tempfile implementation may leave an extra hidden hard link on platforms
+that need a link/unlink fallback or after an interruption. This is not a
+transactional directory-durability guarantee. On Unix the new export has mode 0600. No existing output,
+including a concurrent writer's file or a symlink, is replaced.
+
+An unchanged import rerun refuses the existing output; reuse that snapshot for
+unchanged autoindex reruns. For an updated source or parser version, prepare a
+new export folder and index it under a new prefix/state directory. This version
+does not refresh or remove a previous snapshot. Retain the old snapshot/index
+until the new one has been verified, then use the normal ownership-aware
+autoindex remove workflow. A killed process can leave a hidden sanitized temp
+file; it is not a complete export. Use a new destination after interruption.
+
+## Format evidence and verification boundary
+
+Claude Code documents its on-disk JSONL as an internal format that can change:
+[session storage](https://code.claude.com/docs/en/sessions#where-transcripts-are-stored).
+This adapter supports the explicit shapes above, not every client version,
+`stream-json`, rendered `/export` text, Codex or Cursor logs.
+
+Reference retrieved before implementation: Anthropic's MIT-licensed SDK at
+`0e89b298e771259378a6cd3734e00d323dff5703`,
+[`sessions.py:897–929`](https://github.com/anthropics/claude-agent-sdk-python/blob/0e89b298e771259378a6cd3734e00d323dff5703/src/claude_agent_sdk/_internal/sessions.py#L897-L929)
+parses typed UUID-bearing events, and
+[`sessions.py:1248–1285`](https://github.com/anthropics/claude-agent-sdk-python/blob/0e89b298e771259378a6cd3734e00d323dff5703/src/claude_agent_sdk/_internal/sessions.py#L1248-L1285)
+distinguishes visible messages and session identities. Approach only; no code
+copied. XERJ intentionally refuses malformed input instead of silently skipping
+it, and exports branch flags rather than choosing the SDK's active chain.
+
+All checked-in fixtures are synthetic. No real personal transcript or real
+Claude client release is claimed validated by those fixtures. Unit and CLI
+tests cover redaction, provenance, filtering, size limits, malformed tails,
+unsupported shapes, no-clobber publication and recovery.
diff --git a/tools/session-import/VERIFICATION.md b/tools/session-import/VERIFICATION.md
new file mode 100644
index 000000000..331990332
--- /dev/null
+++ b/tools/session-import/VERIFICATION.md
@@ -0,0 +1,135 @@
+# Local verification — 2026-10-25
+
+OpenAI Codex, operated by Jarvis636431. macOS arm64, Rust 1.97.1.
+Candidate source is recorded in `2f46093b`, independently based on upstream
+`b8c41cb6`. No dependency on the open batch-retrieval benchmark PR #2384.
+The local release binary reports `xerj v1.0.0-rc.99`; this is a source-built
+candidate, not a verification claim about published release assets.
+
+## Commands and observed results
+
+Run from `engine/`, except the end-to-end command marked below:
+
+```sh
+cargo +1.97.1 fmt --all
+cargo +1.97.1 fmt --all -- --check
+cargo +1.97.1 build --locked --offline --release -j 32 \
+ -p xerj-autoindex -p xerj-server -p es-yaml-runner
+
+TMPDIR=/private/tmp cargo +1.97.1 test --locked --offline -p xerj-autoindex -- \
+ --test-threads=1 \
+ --skip content::tests::lossy_display_collisions_keep_distinct_alias_identities \
+ --skip corpus_data_skip_tests::corpus_index_summary_splits_indexed_extensions_from_ignored_data \
+ --skip inventory_delta_tests::a_non_utf8_excluded_name_is_matched_on_disk_via_path_id
+
+TMPDIR=/private/tmp cargo +1.97.1 test --locked --offline \
+ -p xerj-server --test sessions_cli --bin xerj
+
+cargo +1.97.1 clippy --locked --offline --no-deps \
+ -p xerj-server --bin xerj --test sessions_cli -- -D warnings
+cargo +1.97.1 clippy --locked --offline --no-deps \
+ -p xerj-autoindex --all-targets -- -D warnings -A dead_code
+```
+
+- Scoped release build, formatting and whitespace checks passed.
+- Autoindex: **1,839 passed, 0 failed, 5 existing ignored, 3 filtered** across
+ its unit/integration/doc-test invocations. All **11** new session tests passed.
+- Main program: **119 passed**. New CLI integration tests: **3 passed**.
+- Both listed Clippy invocations passed. Autoindex's unmodified
+ `content.rs:143` `resolve_reporting` dead-code warning prevents a blanket
+ `-D warnings` run on this host; this exact failure was separately reproduced
+ on the unmodified upstream baseline. No repository lint allowance was added.
+- Workflow timeout guard passed; no workflows were changed.
+
+The initial sandbox invocation could not bind test loopback sockets; the final
+tests ran with loopback permitted. The normal parallel invocation also hit
+existing process-global/path interference; final full tests are explicitly
+serial with the canonical temporary root above. No test assertions were changed.
+
+### Why the three filters exist
+
+Each named test was run individually on the unmodified, detached `b8c41cb6`
+worktree with the same Rust toolchain and `/private/tmp` root, and failed there:
+
+| Test | Observed baseline failure on this filesystem |
+|---|---|
+| `lossy_display_collisions_keep_distinct_alias_identities` | Creating the fixture's non-UTF-8 filename returns OS error 92, Illegal byte sequence |
+| `a_non_utf8_excluded_name_is_matched_on_disk_via_path_id` | Same non-UTF-8 filename restriction |
+| `corpus_index_summary_splits_indexed_extensions_from_ignored_data` | `rows.csv` and `ROWS.CSV` name the same file on the case-insensitive filesystem; observed csv count 1 versus expected 2 |
+
+These filters are local command arguments, not committed skips or weaker tests.
+The unfiltered serial candidate suite observed **1,792 passed, 3 failed,
+2 ignored** before the final inclusive-limit and FIFO regressions were added. Linux filesystem behavior
+and hosted CI are not inferred from this local result.
+
+## Real CLI, index and query
+
+From the repository root:
+
+```sh
+python3 tools/session-import/smoke.py --binary engine/target/release/xerj
+```
+
+Observed on the final release candidate:
+
+- Offline import exported **2** redacted text turns from the synthetic fixture;
+ existing-output refusal preserved the file.
+- Autoindex dry-run reported **1 file (0 MB)**. The first index and unchanged
+ rerun both completed with exit **0**, no refused datasets/files and **2** live
+ documents. The observed wall times were 0.818 s / 0.353 s respectively; these
+ tiny checks are not a speed benchmark.
+- CLI query `why did we drop polling` retrieved the decision text. Stored
+ metadata retained original `session.jsonl:3` and `src/watch.rs`; raw email,
+ thinking, tool payload and metadata sentinels were absent from stored source.
+- Both private-node shutdowns exited **0**, and restart preserved **2** documents.
+
+The example is lexical text retrieval with `--no-semantic`, no model/reranker,
+no real personal transcript and no real Claude client release under test.
+It is not generated-answer or confidence-calibration evidence.
+
+## Compatibility gate
+
+On a separate fresh private node running the same release candidate:
+
+```sh
+engine/target/release/es-yaml-runner --url "$PRIVATE_NODE_URL" \
+ --dir engine/tests/es-compat-yaml/yaml
+```
+
+Observed: **229 files; 1,444 passed, 0 failed, 10 skipped; exit 0**.
+The unchanged suite policy reported **75 catch-lenient cases**. No runner,
+YAML, catch-lenient policy or server protocol implementation was changed.
+The private conformance node shut down with exit **0**.
+
+## Negative controls and review
+
+| Invariant | Authority / exercised boundary | Result |
+|---|---|---|
+| Redaction precedes every exported text/metadata write | Native PII plugin, unit fixtures covering its five types and CLI stored-source check | Pass; an isolated source-copy mutation bypassing the plugin produced exactly the two intended privacy-test failures |
+| Incomplete input is never a completed snapshot | Malformed later line/tail, unsupported shapes, duplicate UUIDs, mixed session IDs and size refusals | Pass; no output, staging removed, raw payload absent from diagnostics; corrected synthetic input then succeeds |
+| Existing output survives a race after preflight | `persist_noclobber` boundary with an independently created winner | Pass; a separate source-copy mutation using overwrite publication fails that exact regression |
+| Input source and source citation are explicit | Validated UUIDs, file bytes consumed, source-line/byte metadata, change-detection helper | Pass; source mutation refused; original input preserved; source hashes match |
+| Non-file input cannot hang waiting for a writer | Unix FIFO opened nonblocking and rejected by file type | Pass; no writer or timing assumption in the regression |
+| Prepared data survives the existing index lifecycle | Real CLI + private HTTP node, unchanged autoindex rerun and graceful restart | Pass; 2 documents throughout |
+
+The mutation checks compiled **private copies** of the module. Production source
+was never mutated during builds; its normal tests pass. Before implementation,
+the initial scaffold had two expected session-test failures and one existing
+output preservation check passing.
+
+Review against `docs/CONTRIBUTION_REVIEW.md`: effective diff contains only the
+new offline preparation path, existing-command dispatch/help, two existing
+dependencies, tests/fixture and documentation. No existing workflow, JSONL
+sniffer/extractor, index mapping, persisted product format or recovery path is
+replaced. Export schema 1 is a new-file snapshot; future refresh/migration uses
+an explicitly new output and prefix/state directory, not silent overwrite.
+Tempfile no-clobber behavior and the possible extra hidden hard link are
+documented without a cross-platform transactional-durability claim.
+
+Legacy product migrations, optional accelerators, distributed writes, model
+execution, providers and new egress are not applicable: this command only reads
+the explicit source and writes the explicit export. Windows/Linux real-node
+execution, live client-version compatibility, deep secret removal, automatic
+discovery, active-chain reconstruction, incremental refresh, and commit/graph
+association are not claimed. Website generation gates were not run because
+no landing/generated website files changed. Hosted CI must validate composition.
diff --git a/tools/session-import/smoke.py b/tools/session-import/smoke.py
new file mode 100644
index 000000000..2a7b44e32
--- /dev/null
+++ b/tools/session-import/smoke.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""Synthetic session -> redacted JSONL -> autoindex -> cited decision, on a private node."""
+import argparse
+import json
+import os
+from pathlib import Path
+import socket
+import subprocess
+import tempfile
+import time
+import urllib.request
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def run(binary, arguments, env, work, expected=0):
+ result = subprocess.run([str(binary), *arguments], env=env, cwd=work,
+ capture_output=True, text=True, timeout=90)
+ if result.returncode != expected:
+ raise RuntimeError(f'{arguments[0]} exit {result.returncode}: {result.stderr}')
+ return result
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--binary', type=Path, required=True)
+ args = parser.parse_args()
+ binary = args.binary.resolve(strict=True)
+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')
+ and k.lower() not in ('all_proxy', 'http_proxy', 'https_proxy', 'no_proxy')}
+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_AUTH='', XERJ_NUM_CPUS='4')
+ with tempfile.TemporaryDirectory(prefix='xerj-sessions-smoke-') as temp:
+ work = Path(temp)
+ source = work / 'session.jsonl'
+ source.write_bytes((ROOT / 'engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonl').read_bytes())
+ exports = work / 'export'
+ exports.mkdir()
+ output = exports / 'records.jsonl'
+ command = ['sessions', 'import', '--format', 'claude-code', '--input', str(source), '--output', str(output)]
+ imported = json.loads(run(binary, command, env, work).stdout)
+ assert imported['records'] == 2, imported
+ safe = output.read_bytes()
+ assert b'[email protected]' not in safe and b'DO_NOT_EXPORT' not in safe
+ run(binary, command, env, work, expected=1)
+ assert output.read_bytes() == safe
+ print('offline import: 2 redacted text turns; overwrite refused', flush=True)
+
+ ports = []
+ sockets = []
+ for _ in range(2):
+ sock = socket.socket()
+ sock.bind(('127.0.0.1', 0))
+ sockets.append(sock)
+ ports.append(sock.getsockname()[1])
+ for sock in sockets:
+ sock.close()
+ config = work / 'node.toml'
+ config.write_text(f'[server]\nbind_address="127.0.0.1"\nes_compat_port={ports[0]}\n'
+ f'rest_port={ports[1]}\ngrpc_port=0\ndata_dir={json.dumps(str(work / "data"))}\n'
+ '[limits]\ndisk_flood_stage_percent=0\n')
+ url = f'http://127.0.0.1:{ports[0]}'
+ opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+
+ def request(path, body=None):
+ req = urllib.request.Request(url + path,
+ None if body is None else json.dumps(body).encode(),
+ headers={'Content-Type': 'application/json'})
+ with opener.open(req, timeout=10) as response:
+ return json.load(response)
+
+ def start(log):
+ child = subprocess.Popen([str(binary), '--insecure', '-c', str(config)], cwd=work,
+ env=env, stdout=log, stderr=log)
+ deadline = time.monotonic() + 60
+ try:
+ while time.monotonic() < deadline:
+ if child.poll() is not None:
+ raise RuntimeError('private node exited before readiness')
+ try:
+ request('/')
+ return child
+ except OSError:
+ time.sleep(.1)
+ raise RuntimeError('private node readiness timed out')
+ except BaseException:
+ stop(child)
+ raise
+
+ def stop(child):
+ if child.poll() is None:
+ child.terminate()
+ try:
+ child.wait(timeout=20)
+ except subprocess.TimeoutExpired:
+ child.kill()
+ child.wait(timeout=5)
+ raise RuntimeError('private node required SIGKILL')
+ assert child.returncode == 0, child.returncode
+
+ with (work / 'node.log').open('w') as log:
+ child = start(log)
+ try:
+ index = ['autoindex', str(exports), '--prefix', 'session-smoke', '--url', url,
+ '--state-dir', str(work / 'state'), '--no-graph', '--no-semantic',
+ '--workers', '2', '--max-minutes', '0']
+ dry = run(binary, index + ['--dry-run'], env, work)
+ for line in (dry.stdout + dry.stderr).splitlines():
+ if 'autoindex:' in line and 'files (' in line:
+ print(line, flush=True)
+ for generation in range(2):
+ before = time.monotonic()
+ indexed = run(binary, index + ['--progress', 'json'], env, work)
+ done = []
+ for line in (indexed.stdout + indexed.stderr).splitlines():
+ if line.startswith('xerj-done'):
+ print(line, flush=True)
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if isinstance(event, dict) and event.get('bar'):
+ print(event['bar'], flush=True)
+ if isinstance(event, dict) and event.get('event') == 'done':
+ done.append(event)
+ print(line, flush=True)
+ assert len(done) == 1 and done[0]['ok'] is True and done[0]['exit'] == 0, done
+ assert not done[0].get('datasets_refused') and not done[0].get('files_refused'), done
+ count = request('/session-smoke-*/_count')['count']
+ assert count == 2, count
+ print(f'autoindex pass {generation + 1}: {time.monotonic() - before:.3f}s; {count} live documents', flush=True)
+ result = request('/session-smoke-*/_search', {'size': 10, 'query': {'match_all': {}}})
+ stored = result['hits']['hits']
+ assert len(stored) == 2, result
+ assert '[email protected]' not in json.dumps(stored)
+ assert 'DO_NOT_EXPORT' not in json.dumps(stored)
+ decision = next(h['_source'] for h in stored if 'durable journal' in h['_source']['body'])
+ assert decision['source_file'] == 'session.jsonl' and decision['source_line'] == 3, decision
+ assert decision['files'] == ['src/watch.rs'], decision
+ search = run(binary, ['search', '--prefix', 'session-smoke', '--url', url,
+ '--json', 'why did we drop polling'], env, work)
+ assert 'durable journal' in search.stdout, search.stdout
+ print('search: decision text found; original session.jsonl:3 and src/watch.rs retained', flush=True)
+ finally:
+ stop(child)
+ child = start(log)
+ try:
+ assert request('/session-smoke-*/_count')['count'] == 2
+ print('graceful restart: 2 documents preserved', flush=True)
+ finally:
+ stop(child)
+ return 0
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())