Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(importer): replay external conversations (#507)
YANGYU353eaa3fea2b
diff --git a/AGENTS.md b/AGENTS.mdindex 49db94ff..fe8a8d5b 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -178,7 +178,7 @@ crates/├── ai-memory-workstream/ read-only native transcript + launch adapters (`ai-memory run`).└── ai-memory-cli/ `ai-memory` binary entry point + thin HTTP subcommands.evals/ live A/B harness; workspace member, not shipped.-companions/ai-memory-importer/ standalone OMC wiki importer; NOT a root+companions/ai-memory-importer/ standalone OMC + external-conversation importer; NOT a rootworkspace member — build/test it with`--manifest-path companions/ai-memory-importer/Cargo.toml`.hooks/ per-agent lifecycle hook bundles (shell/native).diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fd93572b..f8a4e94b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Added+- The standalone `ai-memory-importer` companion can now replay bounded generic+ external-conversation JSON into the existing observation/consolidation+ pipeline. It is dry-run by default; `--apply` sends one ordered `/hook/batch`+ with the dedicated `external-import` wire identity (stored in core's closed+ `other` bucket), with extension provenance for assistant and system messages.+ Full-envelope validation, client-side credential redaction,+ stable session/event idempotency keys, event/byte caps, and a durable partial+ failure manifest make interrupted imports safely resumable. Product-specific+ ChatGPT/Claude export adapters and watch folders remain out of tree. (#483)+## [1.33.1] - 2026-08-28### Fixeddiff --git a/companions/ai-memory-importer/Cargo.lock b/companions/ai-memory-importer/Cargo.lockindex f995ff01..85cb43fe 100644--- a/companions/ai-memory-importer/Cargo.lock+++ b/companions/ai-memory-importer/Cargo.lock@@ -2,12 +2,22 @@# It is not intended for manual editing.version = 4+[[package]]+name = "aho-corasick"+version = "1.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"+dependencies = [+ "memchr",+]+[[package]]name = "ai-memory-importer"version = "0.1.0"dependencies = ["anyhow","clap",+ "regex","reqwest","serde","serde_json",@@ -182,6 +192,22 @@ version = "1.0.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"+[[package]]+name = "core-foundation"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "core-foundation-sys"+version = "0.8.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"+[[package]]name = "cpufeatures"version = "0.2.17"@@ -421,10 +447,10 @@ dependencies = ["hyper","hyper-util","rustls",+ "rustls-native-certs","tokio","tokio-rustls","tower-service",- "webpki-roots",][[package]]@@ -651,6 +677,12 @@ version = "1.70.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"+[[package]]+name = "openssl-probe"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"+[[package]]name = "percent-encoding"version = "2.3.2"@@ -795,6 +827,35 @@ dependencies = ["getrandom 0.3.4",]+[[package]]+name = "regex"+version = "1.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-automata",+ "regex-syntax",+]++[[package]]+name = "regex-automata"+version = "0.4.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"+[[package]]name = "reqwest"version = "0.12.28"@@ -816,6 +877,7 @@ dependencies = ["pin-project-lite","quinn","rustls",+ "rustls-native-certs","rustls-pki-types","serde","serde_json",@@ -830,7 +892,6 @@ dependencies = ["wasm-bindgen","wasm-bindgen-futures","web-sys",- "webpki-roots",][[package]]@@ -880,6 +941,18 @@ dependencies = ["zeroize",]+[[package]]+name = "rustls-native-certs"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"+dependencies = [+ "openssl-probe",+ "rustls-pki-types",+ "schannel",+ "security-framework",+]+[[package]]name = "rustls-pki-types"version = "1.14.1"@@ -922,6 +995,38 @@ dependencies = ["winapi-util",]+[[package]]+name = "schannel"+version = "0.1.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"+dependencies = [+ "windows-sys 0.61.2",+]++[[package]]+name = "security-framework"+version = "3.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"+dependencies = [+ "bitflags",+ "core-foundation",+ "core-foundation-sys",+ "libc",+ "security-framework-sys",+]++[[package]]+name = "security-framework-sys"+version = "2.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"+dependencies = [+ "core-foundation-sys",+ "libc",+]+[[package]]name = "serde"version = "1.0.228"@@ -1405,15 +1510,6 @@ dependencies = ["wasm-bindgen",]-[[package]]-name = "webpki-roots"-version = "1.0.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"-dependencies = [- "rustls-pki-types",-]-[[package]]name = "winapi-util"version = "0.1.11"diff --git a/companions/ai-memory-importer/Cargo.toml b/companions/ai-memory-importer/Cargo.tomlindex 49b336bc..0fa7d1e4 100644--- a/companions/ai-memory-importer/Cargo.toml+++ b/companions/ai-memory-importer/Cargo.toml@@ -12,6 +12,7 @@ clap = { version = "4", features = ["derive", "env"] }# Native roots, matching the workspace: the importer POSTs to the same server# an operator may front with a private CA (#492).reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] }+regex = "1"serde = { version = "1", features = ["derive"] }serde_json = "1"serde_yaml = "0.9"diff --git a/companions/ai-memory-importer/README.md b/companions/ai-memory-importer/README.mdindex a853984f..c5cd13bd 100644--- a/companions/ai-memory-importer/README.md+++ b/companions/ai-memory-importer/README.md@@ -5,15 +5,52 @@ running ai-memory server. This crate is deliberately isolated from the rootworkspace: its `Cargo.toml` has its own `[workspace]`, uses only crates.iodependencies, and is not included in root `cargo test --workspace`.-## Supported source: OMC wiki directory+## Supported sources-The first importer supports oh-my-claudecode / OMC flat Markdown wiki+### OMC wiki directory++The original importer supports oh-my-claudecode / OMC flat Markdown wikidirectories. It reads only top-level `*.md` files, skips `index.md` and`session-log-*` by default, and writes deterministic destination paths under`omc/<slug>.md`.-Claude memory graph and Qdrant imports are roadmap items only; there are no code-stubs for them in v1.+### Generic external conversations++`external-conversation` accepts one deliberately small interchange format:++```json+{+ "project": "my-project",+ "source": "chatgpt",+ "session_id": "exported-conversation-id",+ "messages": [+ { "role": "user", "content": "What evidence supports this claim?" },+ { "role": "assistant", "content": "The source supports only part of it." }+ ]+}+```++Roles are `system`, `user`, or `assistant`. Product-specific ChatGPT, Claude+Desktop, Markdown, or other export adapters intentionally stay outside this+repository; they only need to emit the generic envelope.++The importer replays the conversation through ai-memory's public hook pipeline:+one `session-start`, the ordered messages, and one `session-end`. User messages+use the canonical `user-prompt` event. Assistant/system messages use the+validated `ai-memory-importer` extension vocabulary so their role and body stay+available to later consolidation. The whole sequence uses `/hook/batch`, whose+inline processing preserves order and does not return until the session page+and other SessionEnd effects have landed.++Imported sessions use the dedicated `agent=external-import` wire identity,+never a live `codex` or `claude-code` identity. Core's tolerant unknown-agent+boundary stores it in the closed `other` bucket, while the SessionStart+observation names `external:<source>` and assistant/system observations retain+extension provenance. A reader can therefore distinguish an explicit import+and its source without adding product-specific agent kinds to core.++Claude memory graph and Qdrant imports remain roadmap items; there are no code+stubs for them.## Safety contract@@ -39,6 +76,23 @@ stubs for them in v1.`pinned`, and `body`. Unknown frontmatter is ignored.- Auth comes only from `AI_MEMORY_AUTH_TOKEN`; there is intentionally no CLItoken argument.+- External conversations are fully parsed, schema-checked, bounded, and+ sanitized before any manifest or HTTP request is created. Unknown fields and+ roles fail closed. Limits are 2 MiB per file, 128 messages, and 1 MiB total+ message content. Oversized individual messages are UTF-8-safely truncated to+ the same durable hook caps: 16 KiB for user prompts and 2,000 bytes for+ extension messages. The dry-run and manifest report the truncation count.+- Conversation bodies receive client-side credential redaction before replay+ and then cross ai-memory's normal server sanitizer as a second boundary.+- Stable session IDs derive from `(workspace, project, source, session_id)`.+ Each replay event has a stable `ingest_key`; a changed transcript gets a new+ terminal generation key, so rerunning an interrupted import is safe and+ appending to an export re-runs SessionEnd after the new messages.+- A live conversation import is one ordered hook batch. If the server accepts+ only a prefix, the manifest is marked failed with its accepted count; rerun+ the same source to resume via the stable keys.+- There is no inbox or watch-folder mode. Import is an explicit one-shot+ operation.## Usage@@ -78,6 +132,39 @@ Options:- `--show-body`: print full page bodies during dry-run.- `--pinned`: pin all imported pages.+## External conversation usage++Dry-run (the default):++```bash+cargo run --manifest-path companions/ai-memory-importer/Cargo.toml -- \+ external-conversation --file /path/to/conversation.json \+ --workspace default+```++Show the sanitized event bodies in the dry-run:++```bash+cargo run --manifest-path companions/ai-memory-importer/Cargo.toml -- \+ external-conversation --file /path/to/conversation.json \+ --workspace default --show-body+```++Live replay:++```bash+AI_MEMORY_AUTH_TOKEN=... \+cargo run --manifest-path companions/ai-memory-importer/Cargo.toml -- \+ external-conversation --file /path/to/conversation.json \+ --workspace default --apply \+ --manifest-out /tmp/conversation-import-manifest.json+```++The envelope's `project` and the CLI's `--workspace` are both explicit. The+destination must already exist unless `--create-destination` is passed. Source+files are read once; adapters should write their completed generic JSON export+before invoking the importer.+## ValidationRun these from the repository root:diff --git a/companions/ai-memory-importer/src/main.rs b/companions/ai-memory-importer/src/main.rsindex e74393e2..6fafa850 100644--- a/companions/ai-memory-importer/src/main.rs+++ b/companions/ai-memory-importer/src/main.rs@@ -1,10 +1,12 @@use std::collections::HashMap;use std::fs;use std::path::{Component, Path, PathBuf};+use std::sync::OnceLock;use std::time::Duration;use anyhow::{Context, Result, anyhow, bail};use clap::{Parser, Subcommand};+use regex::Regex;use reqwest::StatusCode;use serde::{Deserialize, Serialize};use sha2::{Digest, Sha256};@@ -12,7 +14,15 @@ use url::Url;use walkdir::WalkDir;const IMPORT_VERSION: &str = "omc-wiki-v1";+const CONVERSATION_IMPORT_VERSION: &str = "external-conversation-v1";const DEFAULT_SERVER_URL: &str = "http://127.0.0.1:49374";+const EXTERNAL_IMPORT_AGENT: &str = "external-import";+const EXTERNAL_IMPORT_EXTENSION: &str = "ai-memory-importer";+const MAX_CONVERSATION_FILE_BYTES: u64 = 2 * 1024 * 1024;+const MAX_CONVERSATION_MESSAGES: usize = 128;+const MAX_USER_MESSAGE_BYTES: usize = 16 * 1024;+const MAX_EXTENSION_MESSAGE_BYTES: usize = 2_000;+const MAX_CONVERSATION_TOTAL_BYTES: usize = 1024 * 1024;#[derive(Parser, Debug)]#[command(author, version, about = "Optional ai-memory import companion")]@@ -25,6 +35,8 @@ struct Cli {enum Commands {/// Import an oh-my-claudecode / OMC flat markdown wiki directory.OmcWiki(OmcArgs),+ /// Replay a generic external conversation through ai-memory's hook API.+ ExternalConversation(ConversationArgs),}#[derive(Parser, Debug, Clone)]@@ -53,6 +65,115 @@ struct OmcArgs {pinned: bool,}+#[derive(Parser, Debug, Clone)]+struct ConversationArgs {+ /// JSON file containing the generic conversation envelope.+ #[arg(long)]+ file: PathBuf,+ /// Explicit destination workspace. The project comes from the envelope.+ #[arg(long)]+ workspace: Option<String>,+ #[arg(long, env = "AI_MEMORY_SERVER_URL", default_value = DEFAULT_SERVER_URL)]+ server_url: String,+ /// Perform the replay. Without this flag the command only prints a plan.+ #[arg(long)]+ apply: bool,+ /// Durable replay manifest. Required with --apply.+ #[arg(long)]+ manifest_out: Option<PathBuf>,+ /// Permit the hook router to create a missing destination scope.+ #[arg(long)]+ create_destination: bool,+ /// Include sanitized message bodies in dry-run output.+ #[arg(long)]+ show_body: bool,+}++#[derive(Debug, Clone, Deserialize)]+#[serde(deny_unknown_fields)]+struct ConversationEnvelope {+ project: String,+ source: String,+ session_id: String,+ messages: Vec<ConversationMessage>,+}++#[derive(Debug, Clone, Deserialize)]+#[serde(deny_unknown_fields)]+struct ConversationMessage {+ role: ConversationRole,+ content: String,+}++#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+enum ConversationRole {+ System,+ User,+ Assistant,+}++impl ConversationRole {+ const fn as_str(self) -> &'static str {+ match self {+ Self::System => "system",+ Self::User => "user",+ Self::Assistant => "assistant",+ }+ }+}++#[derive(Debug, Clone, Serialize)]+struct PlannedHookEvent {+ index: usize,+ event: String,+ role: Option<ConversationRole>,+ ingest_key: String,+ url: String,+ body: serde_json::Value,+}++#[derive(Debug, Clone)]+struct ConversationPlan {+ source_path: String,+ source_sha256: String,+ source: String,+ external_session_id: String,+ stable_session_id: String,+ transcript_sha256: String,+ workspace: String,+ project: String,+ truncated_messages: usize,+ events: Vec<PlannedHookEvent>,+}++#[derive(Debug, Clone, Serialize, PartialEq, Eq)]+#[serde(rename_all = "kebab-case")]+enum ConversationManifestStatus {+ Planned,+ Imported,+ Failed,+}++#[derive(Debug, Clone, Serialize)]+struct ConversationManifest {+ import_version: String,+ source_path: String,+ source_sha256: String,+ source: String,+ external_session_id: String,+ stable_session_id: String,+ transcript_sha256: String,+ workspace: String,+ project: String,+ planned_events: usize,+ truncated_messages: usize,+ accepted_events: usize,+ status: ConversationManifestStatus,+ #[serde(skip_serializing_if = "Option::is_none")]+ error: Option<String>,+}+#[derive(Debug, Clone)]struct PlannedPage {source_path: String,@@ -134,11 +255,154 @@ impl PageListBody {}}+#[derive(Debug, Serialize)]+struct HookBatchItem {+ url: String,+ body: serde_json::Value,+}++#[derive(Debug, Deserialize)]+struct HookBatchAck {+ accepted: usize,+ #[serde(default)]+ accepted_indices: Option<Vec<usize>>,+ #[serde(default)]+ failed_index: Option<usize>,+}++struct HookBatchDelivery {+ status: StatusCode,+ ack: HookBatchAck,+}++impl HookBatchAck {+ fn accepted_count(&self) -> usize {+ self.accepted_indices+ .as_ref()+ .map_or(self.accepted, Vec::len)+ }++ fn accepted_every_event(&self, expected: usize) -> bool {+ if self.failed_index.is_some() {+ return false;+ }+ match &self.accepted_indices {+ Some(indices) => indices.iter().copied().eq(0..expected),+ None => self.accepted == expected,+ }+ }+}+#[tokio::main]async fn main() -> Result<()> {match Cli::parse().command {Commands::OmcWiki(args) => run_omc(args).await,+ Commands::ExternalConversation(args) => run_conversation(args).await,+ }+}++async fn run_conversation(args: ConversationArgs) -> Result<()> {+ if args.apply {+ if args+ .workspace+ .as_deref()+ .is_none_or(|value| value.trim().is_empty())+ {+ bail!("--apply requires explicit --workspace");+ }+ if args.manifest_out.is_none() {+ bail!("--apply requires --manifest-out <path>");+ }+ }++ let workspace = args+ .workspace+ .clone()+ .unwrap_or_else(|| "DRY-RUN-WORKSPACE".into());+ // Parse, bound, validate, and sanitize the entire source before creating an+ // HTTP client or writing a manifest. A malformed/hostile source therefore+ // cannot produce a partial import.+ let plan = plan_external_conversation(&args.file, &workspace, &args.server_url)?;+ let mut manifest = conversation_manifest(&plan);++ if !args.apply {+ if args.show_body {+ for event in &plan.events {+ println!(+ "--- event {}: {} ({}) ---\n{}",+ event.index,+ event.event,+ event.role.map_or("lifecycle", ConversationRole::as_str),+ serde_json::to_string_pretty(&event.body)?+ );+ }+ }+ if let Some(path) = &args.manifest_out {+ write_conversation_manifest(path, &manifest)?;+ } else {+ println!(+ "dry-run: planned {} ordered hook events ({} truncated messages); no HTTP writes performed",+ plan.events.len(),+ plan.truncated_messages,+ );+ println!(+ "{}:{} -> {}/{} (stable session {})",+ plan.source,+ plan.external_session_id,+ plan.workspace,+ plan.project,+ plan.stable_session_id+ );+ }+ return Ok(());+ }++ let manifest_path = args.manifest_out.as_ref().unwrap();+ write_conversation_manifest(manifest_path, &manifest)?;+ let client = ImportClient::new(&args.server_url)?;+ if let Err(error) = client+ .preflight_project(&plan.workspace, &plan.project, args.create_destination)+ .await+ {+ manifest.status = ConversationManifestStatus::Failed;+ manifest.error = Some(error.to_string());+ write_conversation_manifest(manifest_path, &manifest)?;+ return Err(error);}++ let delivery = match client.post_hook_batch(&plan.events).await {+ Ok(delivery) => delivery,+ Err(error) => {+ manifest.status = ConversationManifestStatus::Failed;+ manifest.error = Some(error.to_string());+ write_conversation_manifest(manifest_path, &manifest)?;+ return Err(error);+ }+ };+ let ack = delivery.ack;+ manifest.accepted_events = ack.accepted_count();+ if !delivery.status.is_success() || !ack.accepted_every_event(plan.events.len()) {+ manifest.status = ConversationManifestStatus::Failed;+ manifest.error = Some(format!(+ "hook batch returned HTTP {} and accepted {} of {} events; failed_index={:?}; rerun the same source to resume safely",+ delivery.status,+ manifest.accepted_events,+ plan.events.len(),+ ack.failed_index+ ));+ write_conversation_manifest(manifest_path, &manifest)?;+ bail!(manifest.error.clone().unwrap());+ }+ manifest.status = ConversationManifestStatus::Imported;+ write_conversation_manifest(manifest_path, &manifest)?;+ println!(+ "import complete: replayed {} events into {}/{} as session {}",+ plan.events.len(),+ plan.workspace,+ plan.project,+ plan.stable_session_id+ );+ Ok(())}async fn run_omc(args: OmcArgs) -> Result<()> {@@ -257,6 +521,432 @@ async fn run_omc(args: OmcArgs) -> Result<()> {Ok(())}+fn plan_external_conversation(+ file: &Path,+ workspace: &str,+ server_url: &str,+) -> Result<ConversationPlan> {+ validate_label(workspace, "workspace", 128)?;+ let metadata = fs::metadata(file)+ .with_context(|| format!("read conversation source {}", file.display()))?;+ if !metadata.is_file() {+ bail!("--file must be a regular file");+ }+ if metadata.len() > MAX_CONVERSATION_FILE_BYTES {+ bail!(+ "conversation file is {} bytes; maximum is {MAX_CONVERSATION_FILE_BYTES}",+ metadata.len()+ );+ }+ let bytes = fs::read(file).with_context(|| format!("read {}", file.display()))?;+ let source_sha256 = sha256_hex(&bytes);+ let mut envelope: ConversationEnvelope =+ serde_json::from_slice(&bytes).context("parse generic conversation JSON")?;++ envelope.project = validate_label(&envelope.project, "project", 128)?.to_owned();+ envelope.source = validate_source_name(&envelope.source)?.to_owned();+ envelope.session_id = validate_label(&envelope.session_id, "session_id", 256)?.to_owned();+ if envelope.messages.is_empty() {+ bail!("conversation must contain at least one message");+ }+ if envelope.messages.len() > MAX_CONVERSATION_MESSAGES {+ bail!(+ "conversation has {} messages; maximum is {MAX_CONVERSATION_MESSAGES}",+ envelope.messages.len()+ );+ }++ let mut total_bytes = 0usize;+ let mut user_messages = 0usize;+ let mut truncated_messages = 0usize;+ for (index, message) in envelope.messages.iter_mut().enumerate() {+ total_bytes = total_bytes+ .checked_add(message.content.len())+ .ok_or_else(|| anyhow!("conversation content size overflow"))?;+ if total_bytes > MAX_CONVERSATION_TOTAL_BYTES {+ bail!(+ "conversation content is {total_bytes} bytes; maximum is {MAX_CONVERSATION_TOTAL_BYTES}"+ );+ }+ message.content = sanitize_external_text(&message.content);+ let per_event_cap = if message.role == ConversationRole::User {+ MAX_USER_MESSAGE_BYTES+ } else {+ MAX_EXTENSION_MESSAGE_BYTES+ };+ if message.content.len() > per_event_cap {+ message.content = truncate_utf8(&message.content, per_event_cap);+ truncated_messages += 1;+ }+ if message.content.trim().is_empty() {+ bail!("message {index} is empty after sanitization");+ }+ if message.role == ConversationRole::User {+ user_messages += 1;+ }+ }+ if user_messages == 0 {+ bail!("conversation must contain at least one user message");+ }++ let stable_session_id = stable_external_session_id(+ workspace,+ &envelope.project,+ &envelope.source,+ &envelope.session_id,+ );+ let transcript_sha256 = conversation_fingerprint(&envelope);+ let mut events = Vec::with_capacity(envelope.messages.len() + 2);+ let start_body = serde_json::json!({+ "session_id": stable_session_id,+ "model": truncate_utf8(&format!("external:{}", envelope.source), 160),+ "external_source": envelope.source,+ "external_session_id": envelope.session_id,+ });+ events.push(planned_hook_event(+ server_url,+ workspace,+ &envelope.project,+ &stable_session_id,+ 0,+ "session-start",+ None,+ None,+ start_body,+ )?);++ for (message_index, message) in envelope.messages.iter().enumerate() {+ let event_index = message_index + 1;+ let (event, source_event, body) = match message.role {+ ConversationRole::User => (+ "user-prompt",+ None,+ serde_json::json!({+ "session_id": stable_session_id,+ "prompt": message.content,+ "external_source": envelope.source,+ "external_role": message.role.as_str(),+ }),+ ),+ ConversationRole::Assistant | ConversationRole::System => {+ let source_event = format!("{}-message", message.role.as_str());+ let title = first_line_title(&message.content, message.role);+ (+ if message.role == ConversationRole::Assistant {+ "external.assistant-message"+ } else {+ "external.system-message"+ },+ Some(source_event),+ serde_json::json!({+ "session_id": stable_session_id,+ "title": title,+ "message": message.content,+ "external_source": envelope.source,+ "external_role": message.role.as_str(),+ }),+ )+ }+ };+ events.push(planned_hook_event(+ server_url,+ workspace,+ &envelope.project,+ &stable_session_id,+ event_index,+ event,+ Some(message.role),+ source_event.as_deref(),+ body,+ )?);+ }++ let end_index = envelope.messages.len() + 1;+ let end_body = serde_json::json!({+ "session_id": stable_session_id,+ "external_source": envelope.source,+ "external_session_id": envelope.session_id,+ "transcript_sha256": transcript_sha256,+ });+ events.push(planned_hook_event(+ server_url,+ workspace,+ &envelope.project,+ &stable_session_id,+ end_index,+ "session-end",+ None,+ None,+ end_body,+ )?);++ Ok(ConversationPlan {+ source_path: file.to_string_lossy().into_owned(),+ source_sha256,+ source: envelope.source,+ external_session_id: envelope.session_id,+ stable_session_id,+ transcript_sha256,+ workspace: workspace.to_owned(),+ project: envelope.project,+ truncated_messages,+ events,+ })+}++#[allow(clippy::too_many_arguments)]+fn planned_hook_event(+ server_url: &str,+ workspace: &str,+ project: &str,+ stable_session_id: &str,+ index: usize,+ event: &str,+ role: Option<ConversationRole>,+ source_event: Option<&str>,+ body: serde_json::Value,+) -> Result<PlannedHookEvent> {+ let body_bytes = serde_json::to_vec(&body)?;+ let ingest_key = event_ingest_key(stable_session_id, index, event, &body_bytes);+ let url = build_hook_url(+ server_url,+ event,+ workspace,+ project,+ stable_session_id,+ &ingest_key,+ source_event,+ )?;+ Ok(PlannedHookEvent {+ index,+ event: event.to_owned(),+ role,+ ingest_key,+ url,+ body,+ })+}++fn build_hook_url(+ server_url: &str,+ event: &str,+ workspace: &str,+ project: &str,+ stable_session_id: &str,+ ingest_key: &str,+ source_event: Option<&str>,+) -> Result<String> {+ let mut url = Url::parse(server_url).context("invalid --server-url")?;+ let base = url.path().trim_end_matches('/');+ url.set_path(&format!("{base}/hook"));+ url.set_fragment(None);+ url.set_query(None);+ {+ let mut query = url.query_pairs_mut();+ query+ .append_pair("event", event)+ .append_pair("agent", EXTERNAL_IMPORT_AGENT)+ .append_pair("workspace", workspace)+ .append_pair("project", project)+ .append_pair("project_src", "marker")+ .append_pair("session_id", stable_session_id)+ .append_pair("ingest_key", ingest_key);+ if let Some(source_event) = source_event {+ query+ .append_pair("extension", EXTERNAL_IMPORT_EXTENSION)+ .append_pair("source_event", source_event);+ }+ }+ Ok(url.into())+}++fn validate_label<'a>(value: &'a str, name: &str, max_bytes: usize) -> Result<&'a str> {+ let trimmed = value.trim();+ if trimmed.is_empty() {+ bail!("{name} must not be empty");+ }+ if trimmed.len() > max_bytes {+ bail!("{name} is {} bytes; maximum is {max_bytes}", trimmed.len());+ }+ if trimmed+ .chars()+ .any(|ch| ch.is_control() || matches!(ch, '/' | '\\'))+ {+ bail!("{name} contains a control character or path separator");+ }+ Ok(trimmed)+}++fn validate_source_name(value: &str) -> Result<&str> {+ let source = validate_label(value, "source", 64)?;+ if !source+ .bytes()+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'))+ {+ bail!("source must be an ASCII identifier using letters, digits, '.', '_', '-' or ':'");+ }+ Ok(source)+}++fn stable_external_session_id(+ workspace: &str,+ project: &str,+ source: &str,+ external_session_id: &str,+) -> String {+ let digest = sha256_parts(&[+ CONVERSATION_IMPORT_VERSION.as_bytes(),+ workspace.as_bytes(),+ project.as_bytes(),+ source.as_bytes(),+ external_session_id.as_bytes(),+ ]);+ format!("external-{}", &digest[..32])+}++fn conversation_fingerprint(envelope: &ConversationEnvelope) -> String {+ let mut hasher = Sha256::new();+ hash_part(&mut hasher, CONVERSATION_IMPORT_VERSION.as_bytes());+ hash_part(&mut hasher, envelope.project.as_bytes());+ hash_part(&mut hasher, envelope.source.as_bytes());+ hash_part(&mut hasher, envelope.session_id.as_bytes());+ for message in &envelope.messages {+ hash_part(&mut hasher, message.role.as_str().as_bytes());+ hash_part(&mut hasher, message.content.as_bytes());+ }+ hex_digest(hasher.finalize())+}++fn event_ingest_key(stable_session_id: &str, index: usize, event: &str, body: &[u8]) -> String {+ sha256_parts(&[+ CONVERSATION_IMPORT_VERSION.as_bytes(),+ stable_session_id.as_bytes(),+ index.to_string().as_bytes(),+ event.as_bytes(),+ body,+ ])+}++fn sha256_parts(parts: &[&[u8]]) -> String {+ let mut hasher = Sha256::new();+ for part in parts {+ hash_part(&mut hasher, part);+ }+ hex_digest(hasher.finalize())+}++fn hash_part(hasher: &mut Sha256, part: &[u8]) {+ hasher.update(part.len().to_le_bytes());+ hasher.update(part);+}++fn hex_digest(digest: impl AsRef<[u8]>) -> String {+ digest+ .as_ref()+ .iter()+ .map(|byte| format!("{byte:02x}"))+ .collect()+}++fn first_line_title(content: &str, role: ConversationRole) -> String {+ let first = content.lines().find(|line| !line.trim().is_empty());+ first.map_or_else(+ || format!("Imported {} message", role.as_str()),+ |line| truncate_utf8(line.trim(), 160),+ )+}++fn truncate_utf8(input: &str, max_bytes: usize) -> String {+ if input.len() <= max_bytes {+ return input.to_owned();+ }+ let mut end = max_bytes;+ while end > 0 && !input.is_char_boundary(end) {+ end -= 1;+ }+ input[..end].to_owned()+}++fn sanitize_external_text(input: &str) -> String {+ let controls_removed: String = input+ .chars()+ .map(|ch| {+ if ch == '\n' || ch == '\r' || ch == '\t' || !ch.is_control() {+ ch+ } else {+ '\u{fffd}'+ }+ })+ .collect();+ secret_patterns()+ .iter()+ .fold(controls_removed, |text, rule| {+ rule.regex.replace_all(&text, rule.replacement).into_owned()+ })+}++struct RedactionRule {+ regex: Regex,+ replacement: &'static str,+}++fn secret_patterns() -> &'static [RedactionRule] {+ static RULES: OnceLock<Vec<RedactionRule>> = OnceLock::new();+ RULES.get_or_init(|| {+ vec![+ RedactionRule {+ regex: Regex::new(+ r"(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----",+ )+ .unwrap(),+ replacement: "[REDACTED PRIVATE KEY]",+ },+ RedactionRule {+ regex: Regex::new(+ r"\b(?:github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16})\b",+ )+ .unwrap(),+ replacement: "[REDACTED CREDENTIAL]",+ },+ RedactionRule {+ regex: Regex::new(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{8,}").unwrap(),+ replacement: "$1[REDACTED]",+ },+ RedactionRule {+ regex: Regex::new(+ r#"(?i)\b(api[_-]?key|access[_-]?token|auth[_-]?token|token|secret|password)\b(\s*[:=]\s*)(?:\"[^\"\r\n]{6,}\"|'[^'\r\n]{6,}'|[^\s,;]{6,})"#,+ )+ .unwrap(),+ replacement: "$1$2[REDACTED]",+ },+ ]+ })+}++fn conversation_manifest(plan: &ConversationPlan) -> ConversationManifest {+ ConversationManifest {+ import_version: CONVERSATION_IMPORT_VERSION.into(),+ source_path: plan.source_path.clone(),+ source_sha256: plan.source_sha256.clone(),+ source: plan.source.clone(),+ external_session_id: plan.external_session_id.clone(),+ stable_session_id: plan.stable_session_id.clone(),+ transcript_sha256: plan.transcript_sha256.clone(),+ workspace: plan.workspace.clone(),+ project: plan.project.clone(),+ planned_events: plan.events.len(),+ truncated_messages: plan.truncated_messages,+ accepted_events: 0,+ status: ConversationManifestStatus::Planned,+ error: None,+ }+}++fn write_conversation_manifest(path: &Path, manifest: &ConversationManifest) -> Result<()> {+ atomic_write(path, serde_json::to_string_pretty(manifest)?.as_bytes())+ .with_context(|| format!("write conversation manifest {}", path.display()))+}+fn planned_entry(page: &PlannedPage) -> ManifestEntry {ManifestEntry {import_version: IMPORT_VERSION.into(),@@ -651,6 +1341,27 @@ impl ImportClient {}Ok(resp.json().await?)}++ async fn post_hook_batch(&self, events: &[PlannedHookEvent]) -> Result<HookBatchDelivery> {+ let items: Vec<_> = events+ .iter()+ .map(|event| HookBatchItem {+ url: event.url.clone(),+ body: event.body.clone(),+ })+ .collect();+ let resp = self+ .request(reqwest::Method::POST, "/hook/batch")+ .json(&items)+ .send()+ .await?;+ let status = resp.status();+ let ack = resp+ .json()+ .await+ .with_context(|| format!("hook batch returned HTTP {status} with an invalid ack"))?;+ Ok(HookBatchDelivery { status, ack })+ }}fn enc(s: &str) -> String {@@ -663,12 +1374,75 @@ fn enc_path(s: &str) -> String {#[cfg(test)]mod tests {use super::*;+ use std::io::{Read, Write};+ use std::net::{TcpListener, TcpStream};+ use std::thread;use tempfile::tempdir;fn write(dir: &Path, name: &str, body: &str) {fs::write(dir.join(name), body).unwrap();}+ fn write_conversation(dir: &Path, body: serde_json::Value) -> PathBuf {+ let path = dir.join("conversation.json");+ fs::write(&path, serde_json::to_vec_pretty(&body).unwrap()).unwrap();+ path+ }++ fn sample_conversation() -> serde_json::Value {+ serde_json::json!({+ "project": "memory-lab",+ "source": "chatgpt",+ "session_id": "source-conversation-42",+ "messages": [+ {"role": "system", "content": "Answer concisely."},+ {"role": "user", "content": "Why is the queue bounded?"},+ {"role": "assistant", "content": "To make backpressure explicit."}+ ]+ })+ }++ fn read_http_request(stream: &mut TcpStream) -> (String, Vec<u8>) {+ let mut bytes = Vec::new();+ let mut buf = [0u8; 4096];+ let header_end = loop {+ let read = stream.read(&mut buf).unwrap();+ assert!(read > 0, "client closed before sending complete headers");+ bytes.extend_from_slice(&buf[..read]);+ if let Some(pos) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {+ break pos + 4;+ }+ };+ let headers = String::from_utf8(bytes[..header_end].to_vec()).unwrap();+ let content_length = headers+ .lines()+ .find_map(|line| {+ let (name, value) = line.split_once(':')?;+ name.eq_ignore_ascii_case("content-length")+ .then(|| value.trim().parse::<usize>().unwrap())+ })+ .unwrap_or(0);+ while bytes.len() - header_end < content_length {+ let read = stream.read(&mut buf).unwrap();+ assert!(read > 0, "client closed before sending complete body");+ bytes.extend_from_slice(&buf[..read]);+ }+ (+ headers,+ bytes[header_end..header_end + content_length].to_vec(),+ )+ }++ fn respond_json(stream: &mut TcpStream, status: &str, body: &str) {+ write!(+ stream,+ "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",+ body.len()+ )+ .unwrap();+ stream.flush().unwrap();+ }+#[test]fn parses_omc_frontmatter() {let parsed = parse_markdown("---\ntitle: T\nkind: rule\ntier: procedural\ntags: [a, b]\npinned: true\nextra: ignored\n---\n# Body\ntext").unwrap();@@ -800,4 +1574,278 @@ mod tests {};assert!(!args.overwrite);}++ #[test]+ fn external_plan_is_ordered_bounded_and_does_not_impersonate_a_live_agent() {+ let td = tempdir().unwrap();+ let file = write_conversation(td.path(), sample_conversation());+ let plan = plan_external_conversation(&file, "default", DEFAULT_SERVER_URL).unwrap();++ assert_eq!(plan.events.len(), 5);+ assert_eq!(plan.events[0].event, "session-start");+ assert_eq!(plan.events[2].event, "user-prompt");+ assert_eq!(plan.events[3].event, "external.assistant-message");+ assert_eq!(plan.events[4].event, "session-end");+ for (index, event) in plan.events.iter().enumerate() {+ assert_eq!(event.index, index);+ assert_eq!(event.ingest_key.len(), 64);+ let url = Url::parse(&event.url).unwrap();+ let query: HashMap<_, _> = url.query_pairs().into_owned().collect();+ assert_eq!(+ query.get("agent").map(String::as_str),+ Some("external-import")+ );+ assert_ne!(query.get("agent").map(String::as_str), Some("codex"));+ assert_ne!(query.get("agent").map(String::as_str), Some("claude-code"));+ }+ let assistant_query: HashMap<_, _> = Url::parse(&plan.events[3].url)+ .unwrap()+ .query_pairs()+ .into_owned()+ .collect();+ assert_eq!(+ assistant_query.get("extension").map(String::as_str),+ Some(EXTERNAL_IMPORT_EXTENSION)+ );+ assert_eq!(+ assistant_query.get("source_event").map(String::as_str),+ Some("assistant-message")+ );+ }++ #[test]+ fn stable_ids_make_replays_idempotent_and_new_generations_reend() {+ let td = tempdir().unwrap();+ let file = write_conversation(td.path(), sample_conversation());+ let first = plan_external_conversation(&file, "default", DEFAULT_SERVER_URL).unwrap();+ let replay = plan_external_conversation(&file, "default", DEFAULT_SERVER_URL).unwrap();+ assert_eq!(first.stable_session_id, replay.stable_session_id);+ assert_eq!(first.transcript_sha256, replay.transcript_sha256);+ assert_eq!(+ first+ .events+ .iter()+ .map(|event| &event.ingest_key)+ .collect::<Vec<_>>(),+ replay+ .events+ .iter()+ .map(|event| &event.ingest_key)+ .collect::<Vec<_>>()+ );++ let mut extended = sample_conversation();+ extended["messages"]+ .as_array_mut()+ .unwrap()+ .push(serde_json::json!({"role": "user", "content": "What changed?"}));+ write_conversation(td.path(), extended);+ let next = plan_external_conversation(&file, "default", DEFAULT_SERVER_URL).unwrap();+ assert_eq!(first.stable_session_id, next.stable_session_id);+ assert_ne!(first.transcript_sha256, next.transcript_sha256);+ assert_eq!(first.events[0].ingest_key, next.events[0].ingest_key);+ assert_ne!(+ first.events.last().unwrap().ingest_key,+ next.events.last().unwrap().ingest_key,+ "a changed transcript needs a fresh SessionEnd generation"+ );+ }++ #[test]+ fn external_text_is_sanitized_before_any_hook_body_is_built() {+ let td = tempdir().unwrap();+ let secret = "github_pat_1234567890abcdefghijklmnop";+ let file = write_conversation(+ td.path(),+ serde_json::json!({+ "project": "p",+ "source": "chatgpt",+ "session_id": "s",+ "messages": [+ {"role": "user", "content": format!("token={secret}\nBearer abcdefghijklmnop")}+ ]+ }),+ );+ let plan = plan_external_conversation(&file, "w", DEFAULT_SERVER_URL).unwrap();+ let serialized = serde_json::to_string(&plan.events).unwrap();+ assert!(!serialized.contains(secret));+ assert!(!serialized.contains("abcdefghijklmnop"));+ assert!(serialized.contains("REDACTED"));+ }++ #[tokio::test]+ async fn malformed_or_hostile_source_fails_before_manifest_or_http() {+ let td = tempdir().unwrap();+ let file = write_conversation(+ td.path(),+ serde_json::json!({+ "project": "p",+ "source": "chatgpt",+ "session_id": "s",+ "messages": [{"role": "tool", "content": "hostile"}],+ "unexpected": true+ }),+ );+ let manifest = td.path().join("manifest.json");+ let error = run_conversation(ConversationArgs {+ file,+ workspace: Some("w".into()),+ // If planning accidentally reaches HTTP setup, this URL is invalid.+ server_url: "not a url".into(),+ apply: true,+ manifest_out: Some(manifest.clone()),+ create_destination: false,+ show_body: false,+ })+ .await+ .unwrap_err();+ assert!(+ error+ .to_string()+ .contains("parse generic conversation JSON")+ );+ assert!(+ !manifest.exists(),+ "invalid input must have zero side effects"+ );+ }++ #[tokio::test]+ async fn live_import_round_trips_preflight_and_one_ordered_hook_batch() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let server = thread::spawn(move || {+ let (mut preflight, _) = listener.accept().unwrap();+ let (headers, body) = read_http_request(&mut preflight);+ assert!(+ headers.starts_with("GET /api/v1/workspaces/default/projects/memory-lab/pages ")+ );+ assert!(body.is_empty());+ respond_json(&mut preflight, "200 OK", "[]");++ let (mut hook, _) = listener.accept().unwrap();+ let (headers, body) = read_http_request(&mut hook);+ assert!(headers.starts_with("POST /hook/batch "));+ let batch: serde_json::Value = serde_json::from_slice(&body).unwrap();+ let items = batch.as_array().unwrap();+ assert_eq!(items.len(), 5);+ assert!(+ items[0]["url"]+ .as_str()+ .unwrap()+ .contains("event=session-start")+ );+ assert!(+ items[2]["url"]+ .as_str()+ .unwrap()+ .contains("event=user-prompt")+ );+ assert!(+ items[4]["url"]+ .as_str()+ .unwrap()+ .contains("event=session-end")+ );+ assert!(items.iter().all(|item| {+ item["url"]+ .as_str()+ .unwrap()+ .contains("agent=external-import")+ }));+ respond_json(&mut hook, "200 OK", r#"{"accepted":5}"#);+ });++ let td = tempdir().unwrap();+ let file = write_conversation(td.path(), sample_conversation());+ let manifest = td.path().join("manifest.json");+ run_conversation(ConversationArgs {+ file,+ workspace: Some("default".into()),+ server_url: format!("http://{address}"),+ apply: true,+ manifest_out: Some(manifest.clone()),+ create_destination: false,+ show_body: false,+ })+ .await+ .unwrap();+ server.join().unwrap();++ let persisted: serde_json::Value =+ serde_json::from_slice(&fs::read(manifest).unwrap()).unwrap();+ assert_eq!(persisted["status"], "imported");+ assert_eq!(persisted["accepted_events"], 5);+ assert_eq!(persisted["planned_events"], 5);+ }++ #[test]+ fn conversation_caps_reject_before_planning_events() {+ let td = tempdir().unwrap();+ let messages: Vec<_> = (0..=MAX_CONVERSATION_MESSAGES)+ .map(|index| serde_json::json!({"role": "user", "content": format!("m{index}")}))+ .collect();+ let file = write_conversation(+ td.path(),+ serde_json::json!({+ "project": "p",+ "source": "chatgpt",+ "session_id": "s",+ "messages": messages+ }),+ );+ let error = plan_external_conversation(&file, "w", DEFAULT_SERVER_URL).unwrap_err();+ assert!(error.to_string().contains("maximum"));+ }++ #[test]+ fn oversized_messages_are_utf8_safely_truncated_to_hook_event_caps() {+ let td = tempdir().unwrap();+ let file = write_conversation(+ td.path(),+ serde_json::json!({+ "project": "p",+ "source": "chatgpt",+ "session_id": "s",+ "messages": [+ {"role": "user", "content": "界".repeat(MAX_USER_MESSAGE_BYTES)},+ {"role": "assistant", "content": "界".repeat(MAX_EXTENSION_MESSAGE_BYTES)}+ ]+ }),+ );+ let plan = plan_external_conversation(&file, "w", DEFAULT_SERVER_URL).unwrap();+ assert_eq!(plan.truncated_messages, 2);+ assert!(plan.events[1].body["prompt"].as_str().unwrap().len() <= MAX_USER_MESSAGE_BYTES);+ assert!(+ plan.events[2].body["message"].as_str().unwrap().len() <= MAX_EXTENSION_MESSAGE_BYTES+ );+ }++ #[test]+ fn batch_ack_requires_every_index_and_no_failure() {+ assert!(+ HookBatchAck {+ accepted: 3,+ accepted_indices: None,+ failed_index: None,+ }+ .accepted_every_event(3)+ );+ assert!(+ HookBatchAck {+ accepted: 0,+ accepted_indices: Some(vec![0, 1, 2]),+ failed_index: None,+ }+ .accepted_every_event(3)+ );+ assert!(+ !HookBatchAck {+ accepted: 1,+ accepted_indices: Some(vec![0, 2]),+ failed_index: Some(1),+ }+ .accepted_every_event(3)+ );+ }}diff --git a/docs/companion-crates.md b/docs/companion-crates.mdindex 68d7ff30..20b5d769 100644--- a/docs/companion-crates.md+++ b/docs/companion-crates.md@@ -73,6 +73,10 @@ every source format and migration workflow.Initial source support is intentionally narrow:- oh-my-claudecode / OMC flat markdown wiki directories.+- generic external-conversation JSON envelopes (`project`, `source`,+ `session_id`, and bounded `role`/`content` messages), replayed through the+ public ordered hook-batch surface. Product-specific export adapters remain+ outside this repository.Future sources can include:@@ -151,6 +155,10 @@ Re-home by kind:preservation in companion imports.- Carry idempotency keys or source fingerprints in companion-side state so failedimports can be resumed safely.+- Validate and sanitize a complete external-conversation envelope before the+ first HTTP request; use a stable imported session identity and per-event+ idempotency keys, and send the dedicated `external-import` wire identity+ rather than impersonating a live coding harness.- Surface all destructive actions in dry-run output before live mode.- Treat non-overwrite checks as best-effort unless/until core exposes a genericcompare-and-write seam; companion v1 re-checks before each write but cannot make