AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(cli): prepare Claude Code session text for autoindex (#2406)
Jarvisc62be98de6bd
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e2374ea3b..2427cf3df 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added+- `xerj sessions import --format claude-code --input FILE --output NEW.jsonl`+ prepares an explicit Claude Code JSONL transcript for ordinary `autoindex`.+ It emits text-bearing user/assistant records through the existing native+ PII plugin before writing, preserving UUIDs, parent links, source filename,+ line/byte offset and explicitly referenced file paths. Tool bodies,+ thinking and metadata records are omitted. The export is an offline,+ new-file snapshot: malformed input, unsupported turn shapes, repeated UUIDs+ or a changed source refuse publication; existing output is never replaced.+ This covers one preparation format for #1793, not automatic session+ discovery, incremental synchronization or universal secret removal. See+ [the import guide](tools/session-import/README.md).+- With `[cluster] enabled = true`, `POST /v1/indices/{name}/docs` (onedocument, or a JSON array with one document per element),`DELETE /v1/indices/{name}/docs/{id}`, anddiff --git a/engine/Cargo.lock b/engine/Cargo.lockindex da00ba696..ff36d778d 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -7459,8 +7459,10 @@ dependencies = ["tree-sitter-swift","tree-sitter-typescript","tree-sitter-zig",+ "uuid","walkdir","xerj-common",+ "xerj-wasm","xxhash-rust","zip 9.0.5",]diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.tomlindex d4353452e..72c409f26 100644--- a/engine/crates/xerj-autoindex/Cargo.toml+++ b/engine/crates/xerj-autoindex/Cargo.toml@@ -10,6 +10,9 @@ description = "xerj autoindex — point it at any folder and make the contents A# The machine-resource policy (cores, memory safe zone, thread priority) is# shared with the server: one policy, one place (#240).xerj-common = { path = "../xerj-common" }+# Session exports use the same native PII redactor as ingest pipelines.+xerj-wasm = { path = "../xerj-wasm" }+uuid.workspace = trueanyhow.workspace = trueserde.workspace = trueserde_json.workspace = truediff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex b9775996d..367943c25 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -49,6 +49,7 @@ mod reconcile_plan;pub mod remove;pub mod resources;pub mod search;+pub mod sessions;pub mod sniff;pub mod source;pub mod staleness;diff --git a/engine/crates/xerj-autoindex/src/sessions.rs b/engine/crates/xerj-autoindex/src/sessions.rsnew file mode 100644index 000000000..8767a7963--- /dev/null+++ b/engine/crates/xerj-autoindex/src/sessions.rs@@ -0,0 +1,697 @@+//! Explicit, offline session export before ordinary autoindex ingestion.+use anyhow::{bail, ensure, Context, Result};+use serde::Serialize;+use serde_json::{json, Value};+use sha2::{Digest, Sha256};+use std::collections::HashSet;+use std::fs::{self, File, Metadata};+use std::io::{BufRead, BufReader, Read, Write};+use std::path::{Path, PathBuf};+use xerj_wasm::{builtins::PiiRedactionPlugin, ProcessAction, TransformPlugin};++const MAX_INPUT_BYTES: u64 = 512 * 1024 * 1024;+const MAX_LINE_BYTES: u64 = 1024 * 1024;+const MAX_TEXT_BYTES: usize = 256 * 1024;+const MAX_RECORDS: usize = 100_000;++pub const HELP: &str = "xerj sessions import — offline session preparation for autoindex++USAGE: xerj sessions import --format claude-code --input FILE --output NEW.jsonl++Read one explicit Claude Code JSONL transcript; write user/assistant text+records through the existing native PII redactor before any output is stored.+Preserve session/message UUIDs, parent UUIDs, source filename/line/byte offset,+branch metadata and file paths explicitly named by Read/Write/Edit/MultiEdit.+Skip tool results, tool inputs except file paths, thinking and metadata.+This exports all text-bearing branches in file order, not an active conversation+chain. Unsupported/malformed turns or repeated message UUIDs refuse the export.++The output is a new, private JSONL snapshot. Existing output is never replaced;+parse failures publish nothing. No node is started or contacted. Review the+export, then run: xerj autoindex <export-folder> --prefix <your-prefix>+The redactor handles email/IP/card/SSN/phone patterns, not every secret or name.+Input <=512 MiB; line <=1 MiB; turn text <=256 KiB; <=100000 message UUIDs.+Exit 0: exported, JSON summary on stdout; 2: usage; 1: import/publication error.+";++#[derive(Debug, Serialize)]+pub struct ImportSummary {+ pub schema_version: u32,+ pub records: u64,+ pub source_lines: u64,+ pub skipped_records: u64,+ pub skipped_blocks: u64,+ pub source_sha256: String,+ pub output_sha256: String,+}++fn uuid_field(entry: &Value, field: &str, line: u64) -> Result<String> {+ let text = entry[field]+ .as_str()+ .filter(|s| s.len() == 36)+ .with_context(|| format!("line {line}: expected UUID in {field}"))?;+ uuid::Uuid::parse_str(text)+ .map(|id| id.to_string())+ .map_err(|_| anyhow::anyhow!("line {line}: invalid UUID in {field}"))+}++fn optional_text(entry: &Value, field: &str, line: u64) -> Result<Value> {+ match entry.get(field) {+ None | Some(Value::Null) => Ok(Value::Null),+ Some(Value::String(s)) => Ok(json!(s)),+ _ => bail!("line {line}: expected text in {field}"),+ }+}++fn flag(entry: &Value, field: &str, line: u64) -> Result<bool> {+ match entry.get(field) {+ None | Some(Value::Null) => Ok(false),+ Some(Value::Bool(value)) => Ok(*value),+ _ => bail!("line {line}: expected boolean in {field}"),+ }+}++fn turn_text(content: &Value, line: u64, skipped: &mut u64) -> Result<(String, Vec<String>)> {+ if let Some(text) = content.as_str() {+ ensure!(+ text.len() <= MAX_TEXT_BYTES,+ "line {line}: turn text exceeds 256 KiB"+ );+ return Ok((text.to_string(), Vec::new()));+ }+ let blocks = content+ .as_array()+ .with_context(|| format!("line {line}: unsupported message content"))?;+ let mut text = String::new();+ let mut files = Vec::new();+ for block in blocks {+ let kind = block["type"]+ .as_str()+ .with_context(|| format!("line {line}: missing content block type"))?;+ if kind == "text" {+ let part = block["text"]+ .as_str()+ .with_context(|| format!("line {line}: malformed text block"))?;+ let separator_bytes = usize::from(!text.is_empty());+ ensure!(+ text.len() + part.len() + separator_bytes <= MAX_TEXT_BYTES,+ "line {line}: turn text exceeds 256 KiB"+ );+ if !text.is_empty() {+ text.push('\n');+ }+ text.push_str(part);+ } else {+ ensure!(+ matches!(+ kind,+ "tool_use"+ | "server_tool_use"+ | "tool_result"+ | "thinking"+ | "redacted_thinking"+ | "image"+ | "document"+ | "container_upload"+ ) || kind.ends_with("_tool_result"),+ "line {line}: unsupported content block type"+ );+ *skipped += 1;+ if kind == "tool_use"+ && matches!(+ block["name"].as_str(),+ Some("Read" | "Write" | "Edit" | "MultiEdit")+ )+ {+ if let Some(path) = block.pointer("/input/file_path") {+ let path = path+ .as_str()+ .with_context(|| format!("line {line}: malformed tool file path"))?;+ if !files.iter().any(|f| f == path) {+ files.push(path.to_string());+ }+ }+ }+ }+ }+ Ok((text, files))+}++fn unchanged(file: &File, before: &Metadata, input: &Path, read: u64) -> Result<()> {+ let after = file.metadata()?;+ let named = fs::metadata(input)?;+ ensure!(+ before.len() == read+ && before.len() == after.len()+ && before.len() == named.len()+ && before.modified()? == after.modified()?+ && before.modified()? == named.modified()?,+ "session input changed during import; use a stopped or archived transcript"+ );+ #[cfg(unix)]+ {+ use std::os::unix::fs::MetadataExt;+ ensure!(+ before.dev() == named.dev() && before.ino() == named.ino(),+ "session input was replaced during import; use an archived transcript"+ );+ }+ Ok(())+}++fn publish(file: tempfile::NamedTempFile, output: &Path) -> Result<()> {+ file.persist_noclobber(output)+ .map_err(|_| anyhow::anyhow!("cannot publish session export: output exists or is not writable; choose a new output path"))?;+ Ok(())+}++pub fn import_claude_code(input: &Path, output: &Path) -> Result<ImportSummary> {+ match fs::symlink_metadata(output) {+ Ok(_) => bail!("session output already exists; choose a new output path"),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => return Err(e).context("cannot inspect session output"),+ }+ let mut options = fs::OpenOptions::new();+ options.read(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::OpenOptionsExt;+ // Opening a FIFO must not block before we can reject non-files.+ options.custom_flags(libc::O_NONBLOCK);+ }+ let file = options.open(input).context("cannot open session input")?;+ let before = file.metadata()?;+ ensure!(before.is_file(), "session input must be a regular file");+ ensure!(+ before.len() <= MAX_INPUT_BYTES,+ "session input exceeds 512 MiB; split or archive a smaller transcript"+ );+ let source_file = input+ .file_name()+ .and_then(|s| s.to_str())+ .context("session input filename must be UTF-8")?;+ let parent = output+ .parent()+ .filter(|p| !p.as_os_str().is_empty())+ .unwrap_or(Path::new("."));+ let mut staging = tempfile::Builder::new()+ .prefix(".xerj-session-")+ .suffix(".tmp")+ .tempfile_in(parent)+ .context("cannot create private export staging file; output parent must exist")?;+ let redactor = PiiRedactionPlugin::new(Vec::new());+ let mut summary = ImportSummary {+ schema_version: 1,+ records: 0,+ source_lines: 0,+ skipped_records: 0,+ skipped_blocks: 0,+ source_sha256: String::new(),+ output_sha256: String::new(),+ };+ let mut source_hash = Sha256::new();+ let mut output_hash = Sha256::new();+ let mut reader = BufReader::new((&file).take(before.len() + 1));+ let mut bytes = Vec::new();+ let mut read = 0;+ let mut written = 0u64;+ let mut session: Option<String> = None;+ let mut seen = HashSet::new();+ loop {+ bytes.clear();+ let count = (&mut reader)+ .take(MAX_LINE_BYTES + 1)+ .read_until(b'\n', &mut bytes)?;+ if count == 0 {+ break;+ }+ summary.source_lines += 1;+ let line = summary.source_lines;+ let offset = read;+ read += count as u64;+ ensure!(+ count as u64 <= MAX_LINE_BYTES,+ "line {line}: session line exceeds 1 MiB"+ );+ source_hash.update(&bytes);+ if bytes.iter().all(u8::is_ascii_whitespace) {+ continue;+ }+ let entry: Value = serde_json::from_slice(&bytes).map_err(|_| {+ anyhow::anyhow!("line {line}: invalid JSON; no session export published")+ })?;+ let kind = entry["type"]+ .as_str()+ .with_context(|| format!("line {line}: missing session record type"))?;+ if !matches!(kind, "user" | "assistant") {+ summary.skipped_records += 1;+ continue;+ }+ let sid = uuid_field(&entry, "sessionId", line)?;+ if let Some(expected) = &session {+ ensure!(+ expected == &sid,+ "line {line}: mixed session IDs are unsupported"+ );+ } else {+ session = Some(sid.clone());+ }+ let mid = uuid_field(&entry, "uuid", line)?;+ ensure!(+ seen.len() < MAX_RECORDS,+ "session exceeds 100000 message UUIDs"+ );+ ensure!(+ seen.insert(mid.clone()),+ "line {line}: repeated message UUID; export a transcript without revisions"+ );+ if flag(&entry, "isMeta", line)? {+ summary.skipped_records += 1;+ continue;+ }+ let message = entry["message"]+ .as_object()+ .with_context(|| format!("line {line}: missing message object"))?;+ ensure!(+ message.get("role").is_none_or(|r| r.as_str() == Some(kind)),+ "line {line}: message role disagrees with record type"+ );+ let (body, files) = turn_text(+ message.get("content").unwrap_or(&Value::Null),+ line,+ &mut summary.skipped_blocks,+ )?;+ if body.trim().is_empty() {+ summary.skipped_records += 1;+ continue;+ }+ let parent_id = if entry["parentUuid"].is_null() {+ Value::Null+ } else {+ json!(uuid_field(&entry, "parentUuid", line)?)+ };+ let timestamp = optional_text(&entry, "timestamp", line)?;+ if let Some(text) = timestamp.as_str() {+ ensure!(+ chrono::DateTime::parse_from_rfc3339(text).is_ok(),+ "line {line}: invalid timestamp"+ );+ }+ let mut record = json!({"body":body, "files":files, "source_file":source_file,+ "cwd":optional_text(&entry,"cwd",line)?, "git_branch":optional_text(&entry,"gitBranch",line)?});+ ensure!(+ redactor.process(&mut record) == ProcessAction::Pass,+ "PII redaction refused session record"+ );+ // Only validated opaque IDs/enums and provenance numbers bypass the+ // text redactor: a phone regex must not corrupt UUID relationships.+ record["id"] = json!(format!("{sid}/{mid}"));+ record["session_schema_version"] = json!(1);+ record["session_format"] = json!("claude-code");+ record["session_id"] = json!(sid);+ record["message_id"] = json!(mid);+ record["parent_id"] = parent_id;+ record["role"] = json!(kind);+ record["timestamp"] = timestamp;+ record["source_line"] = json!(line);+ record["source_byte_offset"] = json!(offset);+ record["is_sidechain"] = json!(flag(&entry, "isSidechain", line)?);+ record["is_summary"] = json!(flag(&entry, "isCompactSummary", line)?);+ let title = record["body"]+ .as_str()+ .unwrap()+ .lines()+ .next()+ .unwrap_or("")+ .chars()+ .take(120)+ .collect::<String>();+ record["title"] = json!(format!("{kind}: {title}"));+ let mut encoded = serde_json::to_vec(&record)?;+ encoded.push(b'\n');+ written += encoded.len() as u64;+ ensure!(+ written <= MAX_INPUT_BYTES,+ "session export exceeds 512 MiB; use a smaller transcript"+ );+ staging.write_all(&encoded)?;+ output_hash.update(&encoded);+ summary.records += 1;+ }+ ensure!(+ summary.records > 0,+ "no supported text-bearing user/assistant turns; no export published"+ );+ unchanged(&file, &before, input, read)?;+ summary.source_sha256 = format!("{:x}", source_hash.finalize());+ summary.output_sha256 = format!("{:x}", output_hash.finalize());+ staging.as_file_mut().sync_all()?;+ publish(staging, output)?;+ Ok(summary)+}++pub fn run_cli(args: &[String]) -> i32 {+ if args.is_empty() || args.iter().any(|a| matches!(a.as_str(), "--help" | "-h")) {+ print!("{HELP}");+ return 0;+ }+ let parsed = (|| -> Result<(PathBuf, PathBuf)> {+ ensure!(args[0] == "import", "expected sessions import");+ let mut input = None;+ let mut output = None;+ let mut format = None;+ let mut it = args[1..].iter();+ while let Some(arg) = it.next() {+ let slot = match arg.as_str() {+ "--input" => &mut input,+ "--output" => &mut output,+ "--format" => &mut format,+ _ => bail!("unknown session import option; see xerj sessions --help"),+ };+ ensure!(slot.is_none(), "duplicate session import option");+ let value = it+ .next()+ .filter(|v| !v.is_empty() && !v.starts_with("--"))+ .context("session import option requires a value")?;+ *slot = Some(value.clone());+ }+ ensure!(+ format.as_deref() == Some("claude-code"),+ "--format claude-code is required"+ );+ Ok((+ PathBuf::from(input.context("--input is required")?),+ PathBuf::from(output.context("--output is required")?),+ ))+ })();+ let (input, output) = match parsed {+ Ok(paths) => paths,+ Err(error) => {+ eprintln!("error: {error}");+ return 2;+ }+ };+ match import_claude_code(&input, &output) {+ Ok(summary) => {+ let mut stdout = std::io::stdout().lock();+ if serde_json::to_writer(&mut stdout, &summary).is_err() || writeln!(stdout).is_err() {+ eprintln!("error: export was published, but its summary could not be written");+ return 1;+ }+ 0+ }+ Err(error) => {+ eprintln!("error: {error:#}");+ 1+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::{json, Value};++ const SESSION: &str = "550e8400-e29b-41d4-a716-446655440000";+ const USER: &str = "550e8400-e29b-41d4-a716-446655440001";+ const ASSISTANT: &str = "550e8400-e29b-41d4-a716-446655440002";++ fn message(role: &str, uuid: &str, content: Value) -> Value {+ json!({"type":role, "uuid":uuid, "sessionId":SESSION,+ "timestamp":"2026-10-25T00:00:00Z",+ "message":{"role":role,"content":content}})+ }++ fn write_input(path: &Path, entries: &[Value]) {+ let mut text = entries+ .iter()+ .map(Value::to_string)+ .collect::<Vec<_>>()+ .join("\n");+ text.push('\n');+ std::fs::write(path, text).unwrap();+ }++ #[test]+ fn sessions_decision_text_is_redacted_and_points_to_the_original_turn() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("conversation.jsonl");+ let output = dir.path().join("safe.jsonl");+ let mut decision = message(+ "assistant",+ ASSISTANT,+ json!([+ {"type":"thinking","thinking":"NEVER_EXPORT_THOUGHTS"},+ {"type":"text","text":"We dropped polling because file events preserve recovery. Contact [email protected]."},+ {"type":"tool_use","name":"Edit","input":{"file_path":"src/watch.rs","old_string":"NEVER_EXPORT_INPUT"}}+ ]),+ );+ decision["parentUuid"] = json!(USER);+ write_input(+ &input,+ &[+ json!({"type":"file-history-snapshot","payload":"NEVER_EXPORT_METADATA"}),+ message("user", USER, json!("Why did we drop polling?")),+ decision,+ ],+ );+ let summary = import_claude_code(&input, &output).unwrap();+ assert_eq!(summary.records, 2);+ let text = std::fs::read_to_string(&output).unwrap();+ assert!(text.contains("[REDACTED_EMAIL]"));+ for secret in [+ "[email protected]",+ "NEVER_EXPORT_THOUGHTS",+ "NEVER_EXPORT_INPUT",+ "NEVER_EXPORT_METADATA",+ ] {+ assert!(!text.contains(secret), "export leaked {secret}");+ }+ let rows: Vec<Value> = text+ .lines()+ .map(|l| serde_json::from_str(l).unwrap())+ .collect();+ assert_eq!(rows[1]["source_line"], 3);+ assert_eq!(rows[1]["message_id"], ASSISTANT);+ assert_eq!(rows[1]["parent_id"], USER);+ assert_eq!(rows[1]["files"], json!(["src/watch.rs"]));+ assert_eq!(+ rows[0].as_object().unwrap().keys().collect::<Vec<_>>(),+ rows[1].as_object().unwrap().keys().collect::<Vec<_>>()+ );+ }++ #[test]+ fn sessions_a_malformed_later_line_publishes_nothing() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ let output = dir.path().join("safe.jsonl");+ write_input(&input, &[message("user", USER, json!("valid text"))]);+ use std::io::Write;+ std::fs::OpenOptions::new()+ .append(true)+ .open(&input)+ .unwrap()+ .write_all(b"{\"secret\":\"NEVER_ECHO_THIS\"")+ .unwrap();+ let error = import_claude_code(&input, &output).unwrap_err().to_string();+ assert!(error.contains("line 2"), "{error}");+ assert!(!error.contains("NEVER_ECHO_THIS"));+ assert!(!output.exists());+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);+ }++ #[test]+ fn sessions_existing_output_is_preserved() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ let output = dir.path().join("safe.jsonl");+ write_input(&input, &[message("user", USER, json!("hello"))]);+ std::fs::write(&output, "existing").unwrap();+ assert!(import_claude_code(&input, &output).is_err());+ assert_eq!(std::fs::read_to_string(&output).unwrap(), "existing");+ }++ #[test]+ fn sessions_redacts_the_five_native_pii_types_and_metadata() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("[email protected]");+ let output = dir.path().join("safe.jsonl");+ let mut entry = message("user", USER, json!("Email: [email protected]. IP: 192.0.2.55. Card: 4111-1111-1111-1111. SSN: 123-45-6789. Phone: 202-555-0147."));+ entry["cwd"] = json!("/home/[email protected]/work");+ entry["gitBranch"] = json!("contact/[email protected]");+ write_input(&input, &[entry]);+ let summary = import_claude_code(&input, &output).unwrap();+ let bytes = std::fs::read(&output).unwrap();+ let text = std::str::from_utf8(&bytes).unwrap();+ for marker in ["EMAIL", "IP", "CC", "SSN", "PHONE"] {+ assert!(text.contains(&format!("[REDACTED_{marker}]")), "{text}");+ }+ assert!(!text.contains("[email protected]"));+ let row: Value = serde_json::from_slice(&bytes).unwrap();+ assert_eq!(row["session_id"], SESSION);+ assert_eq!(row["message_id"], USER);+ assert_eq!(+ summary.source_sha256,+ format!("{:x}", Sha256::digest(std::fs::read(input).unwrap()))+ );+ assert_eq!(+ summary.output_sha256,+ format!("{:x}", Sha256::digest(bytes))+ );+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ assert_eq!(+ std::fs::metadata(output).unwrap().permissions().mode() & 0o777,+ 0o600+ );+ }+ }++ #[test]+ fn sessions_duplicate_mixed_or_unknown_turns_fail_without_partial_output() {+ let first = message("user", USER, json!("first"));+ let second = message("assistant", ASSISTANT, json!("second"));+ let mut mixed = second.clone();+ mixed["sessionId"] = json!("550e8400-e29b-41d4-a716-446655440099");+ let mut unknown = second.clone();+ unknown["message"]["content"] = json!([{"type":"future_text","text":"unsupported"}]);+ let mut bad_role = second.clone();+ bad_role["message"]["role"] = json!("user");+ for (entry, needle) in [+ (first.clone(), "repeated message UUID"),+ (mixed, "mixed session IDs"),+ (unknown, "unsupported content block"),+ (bad_role, "role disagrees"),+ ] {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ let output = dir.path().join("safe.jsonl");+ write_input(&input, &[first.clone(), entry]);+ let error = import_claude_code(&input, &output).unwrap_err().to_string();+ assert!(error.contains(needle), "{error}");+ assert!(!output.exists());+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);+ // Recovery uses the same never-published output after replacing+ // the synthetic malformed input, not weakening a parser guard.+ write_input(&input, &[first.clone(), second.clone()]);+ assert_eq!(import_claude_code(&input, &output).unwrap().records, 2);+ }+ }++ #[test]+ fn sessions_meta_and_tool_only_messages_do_not_become_searchable_text() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ let output = dir.path().join("safe.jsonl");+ let mut meta = message("assistant", ASSISTANT, json!("hidden metadata text"));+ meta["isMeta"] = json!(true);+ write_input(+ &input,+ &[+ message(+ "user",+ USER,+ json!([{"type":"tool_result","content":"raw result"}]),+ ),+ meta,+ ],+ );+ assert!(import_claude_code(&input, &output)+ .unwrap_err()+ .to_string()+ .contains("no supported text-bearing"));+ assert!(!output.exists());+ }++ #[test]+ fn sessions_preserves_branch_flags_and_ids_without_choosing_a_live_leaf() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ let output = dir.path().join("safe.jsonl");+ let mut entry = message("assistant", ASSISTANT, json!("discarded alternative"));+ entry["isSidechain"] = json!(true);+ entry["isCompactSummary"] = json!(true);+ entry["parentUuid"] = json!(USER);+ write_input(&input, &[entry]);+ import_claude_code(&input, &output).unwrap();+ let row: Value = serde_json::from_slice(&std::fs::read(output).unwrap()).unwrap();+ assert_eq!(row["is_sidechain"], true);+ assert_eq!(row["is_summary"], true);+ assert_eq!(row["parent_id"], USER);+ }++ #[test]+ fn sessions_input_change_or_output_creation_cannot_publish_over_existing_state() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ std::fs::write(&input, "before").unwrap();+ let opened = File::open(&input).unwrap();+ let before = opened.metadata().unwrap();+ std::fs::write(&input, "a different length").unwrap();+ assert!(unchanged(&opened, &before, &input, before.len())+ .unwrap_err()+ .to_string()+ .contains("changed during import"));+ let output = dir.path().join("safe.jsonl");+ let mut staging = tempfile::NamedTempFile::new_in(dir.path()).unwrap();+ staging.write_all(b"safe new text").unwrap();+ // Simulate another writer winning after preflight, before publication.+ std::fs::write(&output, "winner").unwrap();+ assert!(publish(staging, &output).is_err());+ assert_eq!(std::fs::read_to_string(output).unwrap(), "winner");+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2);+ }++ #[test]+ fn sessions_limits_refuse_instead_of_truncating_or_falling_back_to_raw_text() {+ for payload in [+ vec![b'x'; MAX_LINE_BYTES as usize + 1],+ message("user", USER, json!("x".repeat(MAX_TEXT_BYTES + 1)))+ .to_string()+ .into_bytes(),+ ] {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ let output = dir.path().join("safe.jsonl");+ std::fs::write(&input, payload).unwrap();+ let error = import_claude_code(&input, &output).unwrap_err().to_string();+ assert!(error.contains("exceeds"), "{error}");+ assert!(!output.exists());+ }+ }++ #[test]+ fn sessions_text_limit_is_inclusive_for_strings_and_block_arrays() {+ for content in [+ json!("x".repeat(MAX_TEXT_BYTES)),+ json!([+ {"type":"text","text":"x".repeat(MAX_TEXT_BYTES)}+ ]),+ ] {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("input.jsonl");+ let output = dir.path().join("safe.jsonl");+ write_input(&input, &[message("user", USER, content)]);+ assert_eq!(import_claude_code(&input, &output).unwrap().records, 1);+ let row: Value = serde_json::from_slice(&std::fs::read(output).unwrap()).unwrap();+ assert_eq!(row["body"].as_str().unwrap().len(), MAX_TEXT_BYTES);+ }+ }++ #[cfg(unix)]+ #[test]+ fn sessions_named_pipe_is_rejected_without_waiting_for_a_writer() {+ use std::os::unix::ffi::OsStrExt;+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("pipe");+ let output = dir.path().join("safe.jsonl");+ let name = std::ffi::CString::new(input.as_os_str().as_bytes()).unwrap();+ // SAFETY: the CString is a valid NUL-terminated temporary path.+ assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o600) }, 0);+ let error = import_claude_code(&input, &output).unwrap_err().to_string();+ assert!(error.contains("regular file"), "{error}");+ assert!(!output.exists());+ }+}diff --git a/engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonl b/engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonlnew file mode 100644index 000000000..05a928653--- /dev/null+++ b/engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonl@@ -0,0 +1,4 @@+{"type":"file-history-snapshot","payload":"DO_NOT_EXPORT_METADATA"}+{"type":"user","sessionId":"550e8400-e29b-41d4-a716-446655440000","uuid":"550e8400-e29b-41d4-a716-446655440001","timestamp":"2026-10-25T00:00:00Z","message":{"role":"user","content":"Why did we drop polling? Contact [email protected] or 202-555-0147."}}+{"type":"assistant","sessionId":"550e8400-e29b-41d4-a716-446655440000","uuid":"550e8400-e29b-41d4-a716-446655440002","parentUuid":"550e8400-e29b-41d4-a716-446655440001","timestamp":"2026-10-25T00:00:01Z","gitBranch":"feature/watch","message":{"role":"assistant","content":[{"type":"thinking","thinking":"DO_NOT_EXPORT_THINKING"},{"type":"text","text":"We dropped polling because file events avoid repeated scans while the durable journal preserves recovery."},{"type":"tool_use","name":"Edit","input":{"file_path":"src/watch.rs","new_string":"DO_NOT_EXPORT_TOOL_INPUT"}}]}}+{"type":"user","sessionId":"550e8400-e29b-41d4-a716-446655440000","uuid":"550e8400-e29b-41d4-a716-446655440003","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"synthetic-tool","content":"DO_NOT_EXPORT_TOOL_OUTPUT"}]}}diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 86c49621b..8d83fec68 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -173,6 +173,7 @@ fn is_top_level_command(arg: &str) -> bool {"index"| "search"| "batch-search"+ | "sessions"| "def"| "init"| "gain"@@ -451,6 +452,7 @@ fn help_text(feedback: bool) -> String {\n\SUBCOMMANDS:\n\\x20\x20\x20\x20xerj batch-search --input FILE ordered batch retrieval, JSON in/out (see xerj batch-search --help)\n\+ \x20\x20\x20\x20xerj sessions import export selected session turns with PII redaction (see xerj sessions --help)\n\\x20\x20\x20\x20xerj index <opts> direct NDJSON → engine ingest (see xerj index --help)\n\\x20\x20\x20\x20xerj autoindex <folder> [opts] zero-config folder discovery + indexing (see xerj autoindex --help)\n\\x20\x20\x20\x20xerj autoindex map [folder] data map of indexed data, or a read-only folder preview\n\@@ -2936,6 +2938,10 @@ fn main() -> Result<()> {let rt = build_runtime(2)?;std::process::exit(rt.block_on(xerj_mcp::run_batch_cli(&args)));}+ if matches!(xerj_common::argv::subcommand().as_deref(), Some("sessions")) {+ let args = xerj_common::argv::args_after_subcommand();+ std::process::exit(xerj_autoindex::sessions::run_cli(&args));+ }// Cores from the resource policy, so `XERJ_NUM_CPUS` and a container CPU// quota reach the async runtime the same way they reach every pool (#240).let cores = xerj_common::resource::cores();diff --git a/engine/crates/xerj-server/tests/sessions_cli.rs b/engine/crates/xerj-server/tests/sessions_cli.rsnew file mode 100644index 000000000..e75bcb59a--- /dev/null+++ b/engine/crates/xerj-server/tests/sessions_cli.rs@@ -0,0 +1,82 @@+//! Offline CLI export: success, redaction and failure publication boundaries.+use serde_json::Value;+use std::path::Path;+use std::process::{Command, Output};++fn import(input: &Path, output: &Path) -> Output {+ Command::new(env!("CARGO_BIN_EXE_xerj"))+ .args(["sessions", "import", "--format", "claude-code", "--input"])+ .arg(input)+ .arg("--output")+ .arg(output)+ .env("XERJ_DISABLE_FEEDBACK", "true")+ .output()+ .unwrap()+}++#[test]+fn sessions_cli_exports_only_redacted_conversation_and_refuses_overwrite() {+ let work = tempfile::tempdir().unwrap();+ let input = work.path().join("session.jsonl");+ std::fs::write(+ &input,+ include_str!("../../xerj-autoindex/tests/fixtures/claude-session.jsonl"),+ )+ .unwrap();+ let output = work.path().join("records.jsonl");+ let result = import(&input, &output);+ assert!(+ result.status.success(),+ "{}",+ String::from_utf8_lossy(&result.stderr)+ );+ let report: Value = serde_json::from_slice(&result.stdout).unwrap();+ assert_eq!(report["schema_version"], 1);+ assert_eq!(report["records"], 2);+ assert_eq!(report["source_lines"], 4);+ assert_eq!(report["skipped_records"], 2);+ assert_eq!(report["skipped_blocks"], 3);+ let bytes = std::fs::read(&output).unwrap();+ let text = std::str::from_utf8(&bytes).unwrap();+ assert!(text.contains("[REDACTED_EMAIL]"));+ assert!(!text.contains("[email protected]"));+ assert!(!text.contains("DO_NOT_EXPORT"));+ let denied = import(&input, &output);+ assert_eq!(denied.status.code(), Some(1));+ assert_eq!(std::fs::read(output).unwrap(), bytes);+}++#[test]+fn sessions_cli_bad_format_is_usage_error_without_output() {+ let work = tempfile::tempdir().unwrap();+ let output = work.path().join("safe.jsonl");+ let result = Command::new(env!("CARGO_BIN_EXE_xerj"))+ .args([+ "sessions", "import", "--format", "unknown", "--input", "unused", "--output",+ ])+ .arg(&output)+ .output()+ .unwrap();+ assert_eq!(result.status.code(), Some(2));+ assert!(!output.exists());+}++#[test]+fn sessions_cli_malformed_tail_keeps_input_and_publishes_no_partial_export() {+ let work = tempfile::tempdir().unwrap();+ let input = work.path().join("session.jsonl");+ let output = work.path().join("records.jsonl");+ let mut bytes =+ include_bytes!("../../xerj-autoindex/tests/fixtures/claude-session.jsonl").to_vec();+ bytes.extend_from_slice(b"{\"private\":\"NEVER_ECHO_PRIVATE_PAYLOAD\"");+ std::fs::write(&input, &bytes).unwrap();+ let result = import(&input, &output);+ assert_eq!(result.status.code(), Some(1));+ assert!(result.stdout.is_empty());+ let error = String::from_utf8_lossy(&result.stderr);+ assert!(error.contains("invalid JSON"), "{error}");+ assert!(!error.contains("NEVER_ECHO_PRIVATE_PAYLOAD"));+ assert!(!output.exists());+ assert_eq!(std::fs::read(input).unwrap(), bytes);+ assert_eq!(std::fs::read_dir(work.path()).unwrap().count(), 1);+}diff --git a/tools/session-import/README.md b/tools/session-import/README.mdnew file mode 100644index 000000000..0049bc0aa--- /dev/null+++ b/tools/session-import/README.md@@ -0,0 +1,148 @@+# Import one Claude Code transcript for decision retrieval++`xerj sessions import` is the explicit preparation step for a single Claude Code+JSONL transcript. It runs locally, calls no model or node, and exports text+turns through XERJ's existing native PII plugin. Review the export, then give its+folder to ordinary `xerj autoindex`. This is the first scoped part of+[#1793](https://github.com/xerj-org/xerj/issues/1793), not automatic transcript+discovery, a multi-client adapter, a live conversation view or incremental sync.++## Try the synthetic example++From the repository root with a built binary and Python 3:++```sh+(cd engine && cargo build --locked --release -j 32 -p xerj-server)+python3 tools/session-import/smoke.py --binary engine/target/release/xerj+```++The example uses only the committed synthetic transcript, prepares a private+export, starts a private loopback node on unused ports, dry-runs autoindex,+indexes and queries the decision, repeats indexing to check idempotence, then+checks graceful restart. It removes its own temporary data at exit. It neither+contacts nor removes any existing node/index. The node runs in default lexical+mode; indexing explicitly uses `--no-semantic`. No semantic-quality claim is made.++## Import your own archived transcript++Choose a stopped/archived file explicitly; this command does not inspect your+home directory or discover other sessions. Use a private destination directory+whose parent you control. The output file must not exist, even if empty.++```sh+set -eu+SESSION_EXPORT=$(mktemp -d)+xerj sessions import --format claude-code \+ --input /path/to/archived-session.jsonl \+ --output "$SESSION_EXPORT/records.jsonl"+```++Success prints a JSON summary containing the schema version, exported-record+count, physical source-line count, skipped-record count, skipped-block count,+and SHA-256 of the source bytes consumed and the exact exported bytes. Blank+lines count as source lines but not skipped records. Skipped blocks count the+non-text blocks inspected in non-meta user/assistant messages. Skipped records+count other top-level records, `isMeta` messages and messages with no text.++The redactor handles the existing email, IP, credit-card, US SSN and phone+patterns. It does not remove every credential, person name, repository path or+private fact. Check the exported JSONL before indexing it. Raw tool payloads,+tool results, thinking and arbitrary message metadata are excluded, including+fields the importer does not recognize. The importer does not change the input.++After review, against your running node:++```sh+set -eu+xerj autoindex "$SESSION_EXPORT" --prefix session-history --dry-run+# Check the estimate and prefix ownership before indexing.+xerj autoindex "$SESSION_EXPORT" --prefix session-history+xerj autoindex map --prefix session-history+xerj search --prefix session-history "why did we drop polling"+```++Only put prepared exports in that folder. Ordinary `autoindex` still accepts+raw JSONL; pointing it at the original transcript folder does **not** activate+this preparation or redaction. Use a prefix you own; follow the normal+autoindex estimate/approval contract for large exports.++## Record and source contract++One output line is one text-bearing `user` or `assistant` event. `message.content`+may be a string or an array of `text`/recognized non-text blocks. Text blocks+retain their order. Each output object has the same key set:++| Fields | Meaning |+|---|---|+| `session_schema_version`, `session_format` | Export schema 1, `claude-code` |+| `id`, `session_id`, `message_id` | Canonical validated UUIDs; ID is session/message UUID |+| `parent_id` | Explicit `parentUuid`, or null; it may reference a skipped event |+| `role`, `title`, `body` | Event role and redacted text; title is the first text line, capped at 120 characters |+| `source_file`, `source_line`, `source_byte_offset` | Redacted original basename, 1-based physical line, 0-based byte offset |+| `files` | Redacted `file_path` values explicitly requested by Read/Write/Edit/MultiEdit blocks in this event |+| `cwd`, `git_branch`, `timestamp` | Explicit metadata or null; timestamp must be RFC 3339 |+| `is_sidechain`, `is_summary` | Explicit branch/compaction flags, default false |++File paths are references in a tool request, not proof that a change succeeded.+No files are opened through these references, and no commit is inferred from+prose, shell commands or branch names. File references in tool-only events are+not exported in this text-only version. Autoindex citations still identify the+prepared file/record; the `source_*` fields locate the original event, whose+byte offset is meaningful only for that archived source digest. Redaction may+change the original filename/path spelling. Query the stored record with+`_source: true` to read all source metadata.++The importer preserves all text-bearing branches in file order. It does not+reconstruct the active conversation, merge forks or claim that a discarded+alternative is current. Compaction summaries are marked, not treated as original+turns. Non-text attachments are not OCRed, and tool results are not searched.++## Refusals, reruns and limits++Exit 0 is a complete export; 2 is invalid CLI usage; 1 is an import or+publication/reporting error. If stdout fails after publication, stderr explicitly+says the export was published: inspect and reuse that complete file rather than+expecting a second import to overwrite it. Diagnostics name the line and schema problem, without+echoing the source payload. Invalid JSON (including a truncated tail), UUIDs,+roles, timestamps or flags, unsupported content blocks, mixed session IDs,+repeated message UUIDs and a changed source refuse the whole export. There is+no fallback to unredacted generic JSONL/text. Metadata-only input also refuses.++Input and output are capped at 512 MiB each, source lines at 1 MiB including+newline, collected turn text at 256 KiB and message UUIDs at 100,000. These are+admission limits, not a measured peak-memory claim. Parsing is line by line.+The temporary output contains only redacted, allowlisted records, is hidden+and is removed on ordinary failure. It is synced before no-clobber publication;+the tempfile implementation may leave an extra hidden hard link on platforms+that need a link/unlink fallback or after an interruption. This is not a+transactional directory-durability guarantee. On Unix the new export has mode 0600. No existing output,+including a concurrent writer's file or a symlink, is replaced.++An unchanged import rerun refuses the existing output; reuse that snapshot for+unchanged autoindex reruns. For an updated source or parser version, prepare a+new export folder and index it under a new prefix/state directory. This version+does not refresh or remove a previous snapshot. Retain the old snapshot/index+until the new one has been verified, then use the normal ownership-aware+autoindex remove workflow. A killed process can leave a hidden sanitized temp+file; it is not a complete export. Use a new destination after interruption.++## Format evidence and verification boundary++Claude Code documents its on-disk JSONL as an internal format that can change:+[session storage](https://code.claude.com/docs/en/sessions#where-transcripts-are-stored).+This adapter supports the explicit shapes above, not every client version,+`stream-json`, rendered `/export` text, Codex or Cursor logs.++Reference retrieved before implementation: Anthropic's MIT-licensed SDK at+`0e89b298e771259378a6cd3734e00d323dff5703`,+[`sessions.py:897–929`](https://github.com/anthropics/claude-agent-sdk-python/blob/0e89b298e771259378a6cd3734e00d323dff5703/src/claude_agent_sdk/_internal/sessions.py#L897-L929)+parses typed UUID-bearing events, and+[`sessions.py:1248–1285`](https://github.com/anthropics/claude-agent-sdk-python/blob/0e89b298e771259378a6cd3734e00d323dff5703/src/claude_agent_sdk/_internal/sessions.py#L1248-L1285)+distinguishes visible messages and session identities. Approach only; no code+copied. XERJ intentionally refuses malformed input instead of silently skipping+it, and exports branch flags rather than choosing the SDK's active chain.++All checked-in fixtures are synthetic. No real personal transcript or real+Claude client release is claimed validated by those fixtures. Unit and CLI+tests cover redaction, provenance, filtering, size limits, malformed tails,+unsupported shapes, no-clobber publication and recovery.diff --git a/tools/session-import/VERIFICATION.md b/tools/session-import/VERIFICATION.mdnew file mode 100644index 000000000..331990332--- /dev/null+++ b/tools/session-import/VERIFICATION.md@@ -0,0 +1,135 @@+# Local verification — 2026-10-25++OpenAI Codex, operated by Jarvis636431. macOS arm64, Rust 1.97.1.+Candidate source is recorded in `2f46093b`, independently based on upstream+`b8c41cb6`. No dependency on the open batch-retrieval benchmark PR #2384.+The local release binary reports `xerj v1.0.0-rc.99`; this is a source-built+candidate, not a verification claim about published release assets.++## Commands and observed results++Run from `engine/`, except the end-to-end command marked below:++```sh+cargo +1.97.1 fmt --all+cargo +1.97.1 fmt --all -- --check+cargo +1.97.1 build --locked --offline --release -j 32 \+ -p xerj-autoindex -p xerj-server -p es-yaml-runner++TMPDIR=/private/tmp cargo +1.97.1 test --locked --offline -p xerj-autoindex -- \+ --test-threads=1 \+ --skip content::tests::lossy_display_collisions_keep_distinct_alias_identities \+ --skip corpus_data_skip_tests::corpus_index_summary_splits_indexed_extensions_from_ignored_data \+ --skip inventory_delta_tests::a_non_utf8_excluded_name_is_matched_on_disk_via_path_id++TMPDIR=/private/tmp cargo +1.97.1 test --locked --offline \+ -p xerj-server --test sessions_cli --bin xerj++cargo +1.97.1 clippy --locked --offline --no-deps \+ -p xerj-server --bin xerj --test sessions_cli -- -D warnings+cargo +1.97.1 clippy --locked --offline --no-deps \+ -p xerj-autoindex --all-targets -- -D warnings -A dead_code+```++- Scoped release build, formatting and whitespace checks passed.+- Autoindex: **1,839 passed, 0 failed, 5 existing ignored, 3 filtered** across+ its unit/integration/doc-test invocations. All **11** new session tests passed.+- Main program: **119 passed**. New CLI integration tests: **3 passed**.+- Both listed Clippy invocations passed. Autoindex's unmodified+ `content.rs:143` `resolve_reporting` dead-code warning prevents a blanket+ `-D warnings` run on this host; this exact failure was separately reproduced+ on the unmodified upstream baseline. No repository lint allowance was added.+- Workflow timeout guard passed; no workflows were changed.++The initial sandbox invocation could not bind test loopback sockets; the final+tests ran with loopback permitted. The normal parallel invocation also hit+existing process-global/path interference; final full tests are explicitly+serial with the canonical temporary root above. No test assertions were changed.++### Why the three filters exist++Each named test was run individually on the unmodified, detached `b8c41cb6`+worktree with the same Rust toolchain and `/private/tmp` root, and failed there:++| Test | Observed baseline failure on this filesystem |+|---|---|+| `lossy_display_collisions_keep_distinct_alias_identities` | Creating the fixture's non-UTF-8 filename returns OS error 92, Illegal byte sequence |+| `a_non_utf8_excluded_name_is_matched_on_disk_via_path_id` | Same non-UTF-8 filename restriction |+| `corpus_index_summary_splits_indexed_extensions_from_ignored_data` | `rows.csv` and `ROWS.CSV` name the same file on the case-insensitive filesystem; observed csv count 1 versus expected 2 |++These filters are local command arguments, not committed skips or weaker tests.+The unfiltered serial candidate suite observed **1,792 passed, 3 failed,+2 ignored** before the final inclusive-limit and FIFO regressions were added. Linux filesystem behavior+and hosted CI are not inferred from this local result.++## Real CLI, index and query++From the repository root:++```sh+python3 tools/session-import/smoke.py --binary engine/target/release/xerj+```++Observed on the final release candidate:++- Offline import exported **2** redacted text turns from the synthetic fixture;+ existing-output refusal preserved the file.+- Autoindex dry-run reported **1 file (0 MB)**. The first index and unchanged+ rerun both completed with exit **0**, no refused datasets/files and **2** live+ documents. The observed wall times were 0.818 s / 0.353 s respectively; these+ tiny checks are not a speed benchmark.+- CLI query `why did we drop polling` retrieved the decision text. Stored+ metadata retained original `session.jsonl:3` and `src/watch.rs`; raw email,+ thinking, tool payload and metadata sentinels were absent from stored source.+- Both private-node shutdowns exited **0**, and restart preserved **2** documents.++The example is lexical text retrieval with `--no-semantic`, no model/reranker,+no real personal transcript and no real Claude client release under test.+It is not generated-answer or confidence-calibration evidence.++## Compatibility gate++On a separate fresh private node running the same release candidate:++```sh+engine/target/release/es-yaml-runner --url "$PRIVATE_NODE_URL" \+ --dir engine/tests/es-compat-yaml/yaml+```++Observed: **229 files; 1,444 passed, 0 failed, 10 skipped; exit 0**.+The unchanged suite policy reported **75 catch-lenient cases**. No runner,+YAML, catch-lenient policy or server protocol implementation was changed.+The private conformance node shut down with exit **0**.++## Negative controls and review++| Invariant | Authority / exercised boundary | Result |+|---|---|---|+| Redaction precedes every exported text/metadata write | Native PII plugin, unit fixtures covering its five types and CLI stored-source check | Pass; an isolated source-copy mutation bypassing the plugin produced exactly the two intended privacy-test failures |+| Incomplete input is never a completed snapshot | Malformed later line/tail, unsupported shapes, duplicate UUIDs, mixed session IDs and size refusals | Pass; no output, staging removed, raw payload absent from diagnostics; corrected synthetic input then succeeds |+| Existing output survives a race after preflight | `persist_noclobber` boundary with an independently created winner | Pass; a separate source-copy mutation using overwrite publication fails that exact regression |+| Input source and source citation are explicit | Validated UUIDs, file bytes consumed, source-line/byte metadata, change-detection helper | Pass; source mutation refused; original input preserved; source hashes match |+| Non-file input cannot hang waiting for a writer | Unix FIFO opened nonblocking and rejected by file type | Pass; no writer or timing assumption in the regression |+| Prepared data survives the existing index lifecycle | Real CLI + private HTTP node, unchanged autoindex rerun and graceful restart | Pass; 2 documents throughout |++The mutation checks compiled **private copies** of the module. Production source+was never mutated during builds; its normal tests pass. Before implementation,+the initial scaffold had two expected session-test failures and one existing+output preservation check passing.++Review against `docs/CONTRIBUTION_REVIEW.md`: effective diff contains only the+new offline preparation path, existing-command dispatch/help, two existing+dependencies, tests/fixture and documentation. No existing workflow, JSONL+sniffer/extractor, index mapping, persisted product format or recovery path is+replaced. Export schema 1 is a new-file snapshot; future refresh/migration uses+an explicitly new output and prefix/state directory, not silent overwrite.+Tempfile no-clobber behavior and the possible extra hidden hard link are+documented without a cross-platform transactional-durability claim.++Legacy product migrations, optional accelerators, distributed writes, model+execution, providers and new egress are not applicable: this command only reads+the explicit source and writes the explicit export. Windows/Linux real-node+execution, live client-version compatibility, deep secret removal, automatic+discovery, active-chain reconstruction, incremental refresh, and commit/graph+association are not claimed. Website generation gates were not run because+no landing/generated website files changed. Hosted CI must validate composition.diff --git a/tools/session-import/smoke.py b/tools/session-import/smoke.pynew file mode 100644index 000000000..2a7b44e32--- /dev/null+++ b/tools/session-import/smoke.py@@ -0,0 +1,155 @@+#!/usr/bin/env python3+"""Synthetic session -> redacted JSONL -> autoindex -> cited decision, on a private node."""+import argparse+import json+import os+from pathlib import Path+import socket+import subprocess+import tempfile+import time+import urllib.request++ROOT = Path(__file__).resolve().parents[2]+++def run(binary, arguments, env, work, expected=0):+ result = subprocess.run([str(binary), *arguments], env=env, cwd=work,+ capture_output=True, text=True, timeout=90)+ if result.returncode != expected:+ raise RuntimeError(f'{arguments[0]} exit {result.returncode}: {result.stderr}')+ return result+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--binary', type=Path, required=True)+ args = parser.parse_args()+ binary = args.binary.resolve(strict=True)+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')+ and k.lower() not in ('all_proxy', 'http_proxy', 'https_proxy', 'no_proxy')}+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_AUTH='', XERJ_NUM_CPUS='4')+ with tempfile.TemporaryDirectory(prefix='xerj-sessions-smoke-') as temp:+ work = Path(temp)+ source = work / 'session.jsonl'+ source.write_bytes((ROOT / 'engine/crates/xerj-autoindex/tests/fixtures/claude-session.jsonl').read_bytes())+ exports = work / 'export'+ exports.mkdir()+ output = exports / 'records.jsonl'+ command = ['sessions', 'import', '--format', 'claude-code', '--input', str(source), '--output', str(output)]+ imported = json.loads(run(binary, command, env, work).stdout)+ assert imported['records'] == 2, imported+ safe = output.read_bytes()+ assert b'[email protected]' not in safe and b'DO_NOT_EXPORT' not in safe+ run(binary, command, env, work, expected=1)+ assert output.read_bytes() == safe+ print('offline import: 2 redacted text turns; overwrite refused', flush=True)++ ports = []+ sockets = []+ for _ in range(2):+ sock = socket.socket()+ sock.bind(('127.0.0.1', 0))+ sockets.append(sock)+ ports.append(sock.getsockname()[1])+ for sock in sockets:+ sock.close()+ config = work / 'node.toml'+ config.write_text(f'[server]\nbind_address="127.0.0.1"\nes_compat_port={ports[0]}\n'+ f'rest_port={ports[1]}\ngrpc_port=0\ndata_dir={json.dumps(str(work / "data"))}\n'+ '[limits]\ndisk_flood_stage_percent=0\n')+ url = f'http://127.0.0.1:{ports[0]}'+ opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))++ def request(path, body=None):+ req = urllib.request.Request(url + path,+ None if body is None else json.dumps(body).encode(),+ headers={'Content-Type': 'application/json'})+ with opener.open(req, timeout=10) as response:+ return json.load(response)++ def start(log):+ child = subprocess.Popen([str(binary), '--insecure', '-c', str(config)], cwd=work,+ env=env, stdout=log, stderr=log)+ deadline = time.monotonic() + 60+ try:+ while time.monotonic() < deadline:+ if child.poll() is not None:+ raise RuntimeError('private node exited before readiness')+ try:+ request('/')+ return child+ except OSError:+ time.sleep(.1)+ raise RuntimeError('private node readiness timed out')+ except BaseException:+ stop(child)+ raise++ def stop(child):+ if child.poll() is None:+ child.terminate()+ try:+ child.wait(timeout=20)+ except subprocess.TimeoutExpired:+ child.kill()+ child.wait(timeout=5)+ raise RuntimeError('private node required SIGKILL')+ assert child.returncode == 0, child.returncode++ with (work / 'node.log').open('w') as log:+ child = start(log)+ try:+ index = ['autoindex', str(exports), '--prefix', 'session-smoke', '--url', url,+ '--state-dir', str(work / 'state'), '--no-graph', '--no-semantic',+ '--workers', '2', '--max-minutes', '0']+ dry = run(binary, index + ['--dry-run'], env, work)+ for line in (dry.stdout + dry.stderr).splitlines():+ if 'autoindex:' in line and 'files (' in line:+ print(line, flush=True)+ for generation in range(2):+ before = time.monotonic()+ indexed = run(binary, index + ['--progress', 'json'], env, work)+ done = []+ for line in (indexed.stdout + indexed.stderr).splitlines():+ if line.startswith('xerj-done'):+ print(line, flush=True)+ try:+ event = json.loads(line)+ except json.JSONDecodeError:+ continue+ if isinstance(event, dict) and event.get('bar'):+ print(event['bar'], flush=True)+ if isinstance(event, dict) and event.get('event') == 'done':+ done.append(event)+ print(line, flush=True)+ assert len(done) == 1 and done[0]['ok'] is True and done[0]['exit'] == 0, done+ assert not done[0].get('datasets_refused') and not done[0].get('files_refused'), done+ count = request('/session-smoke-*/_count')['count']+ assert count == 2, count+ print(f'autoindex pass {generation + 1}: {time.monotonic() - before:.3f}s; {count} live documents', flush=True)+ result = request('/session-smoke-*/_search', {'size': 10, 'query': {'match_all': {}}})+ stored = result['hits']['hits']+ assert len(stored) == 2, result+ assert '[email protected]' not in json.dumps(stored)+ assert 'DO_NOT_EXPORT' not in json.dumps(stored)+ decision = next(h['_source'] for h in stored if 'durable journal' in h['_source']['body'])+ assert decision['source_file'] == 'session.jsonl' and decision['source_line'] == 3, decision+ assert decision['files'] == ['src/watch.rs'], decision+ search = run(binary, ['search', '--prefix', 'session-smoke', '--url', url,+ '--json', 'why did we drop polling'], env, work)+ assert 'durable journal' in search.stdout, search.stdout+ print('search: decision text found; original session.jsonl:3 and src/watch.rs retained', flush=True)+ finally:+ stop(child)+ child = start(log)+ try:+ assert request('/session-smoke-*/_count')['count'] == 2+ print('graceful restart: 2 documents preserved', flush=True)+ finally:+ stop(child)+ return 0+++if __name__ == '__main__':+ raise SystemExit(main())