Lightweight coding agent that runs in your terminal
Rust
Preserve legacy semantics during rollout migration (#37191)
Owen Linaac9f842473a
diff --git a/codex-rs/rollout/src/reverse_jsonl_scanner.rs b/codex-rs/rollout/src/reverse_jsonl_scanner.rsindex 71c5880f9a..70919a7e61 100644--- a/codex-rs/rollout/src/reverse_jsonl_scanner.rs+++ b/codex-rs/rollout/src/reverse_jsonl_scanner.rs@@ -23,6 +23,8 @@ pub struct ReverseJsonlScanner<R> {chunk_position: usize,chunk: Vec<u8>,record_reversed: Vec<u8>,+ max_record_bytes: Option<usize>,+ discarding_oversized_record: bool,}impl<R> ReverseJsonlScanner<R>@@ -52,9 +54,17 @@ wherechunk_position: 0,chunk: vec![0; READ_CHUNK_SIZE],record_reversed: Vec::new(),+ max_record_bytes: None,+ discarding_oversized_record: false,})}+ /// Skips records larger than the configured limit without buffering or parsing them.+ pub fn with_max_record_bytes(mut self, max_record_bytes: usize) -> Self {+ self.max_record_bytes = Some(max_record_bytes);+ self+ }+/// Scans the next nonblank record.////// I/O failures are returned as [`Err`]. Invalid JSON records are returned as@@ -66,6 +76,10 @@ whereloop {if self.chunk_position == 0 {if self.next_chunk_end == 0 {+ if self.discarding_oversized_record {+ self.discarding_oversized_record = false;+ return Ok(None);+ }return Ok(self.finish_record());}@@ -79,14 +93,36 @@ wherelet chunk = &self.chunk[..self.chunk_position];if let Some(newline_position) = chunk.iter().rposition(|byte| *byte == b'\n') {- self.record_reversed- .extend(chunk[newline_position + 1..].iter().rev().copied());+ let fragment = &chunk[newline_position + 1..];+ if !self.discarding_oversized_record {+ if self.max_record_bytes.is_some_and(|max_record_bytes| {+ self.record_reversed.len().saturating_add(fragment.len()) > max_record_bytes+ }) {+ self.record_reversed.clear();+ self.discarding_oversized_record = true;+ } else {+ self.record_reversed.extend(fragment.iter().rev().copied());+ }+ }self.chunk_position = newline_position;+ if self.discarding_oversized_record {+ self.discarding_oversized_record = false;+ continue;+ }if let Some(outcome) = self.finish_record() {return Ok(Some(outcome));}} else {- self.record_reversed.extend(chunk.iter().rev().copied());+ if !self.discarding_oversized_record {+ if self.max_record_bytes.is_some_and(|max_record_bytes| {+ self.record_reversed.len().saturating_add(chunk.len()) > max_record_bytes+ }) {+ self.record_reversed.clear();+ self.discarding_oversized_record = true;+ } else {+ self.record_reversed.extend(chunk.iter().rev().copied());+ }+ }self.chunk_position = 0;}}diff --git a/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rs b/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rsindex 72c471f5de..044fd68e7c 100644--- a/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rs+++ b/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rs@@ -68,6 +68,21 @@ not-jsonOk(())}+#[test]+fn skips_records_over_the_configured_limit() -> std::io::Result<()> {+ let oversized = record(&"x".repeat(128));+ let input = format!(+ "{}\n{}\n{}\n",+ serde_json::to_string(&record("first"))?,+ serde_json::to_string(&oversized)?,+ serde_json::to_string(&record("third"))?+ );+ let mut scanner = ReverseJsonlScanner::new(Cursor::new(input.into_bytes()))?+ .with_max_record_bytes(/*max_record_bytes*/ 32);++ assert_records(&mut scanner, &["third", "first"])+}+#[test]fn accepts_valid_json_at_eof() -> std::io::Result<()> {let input = b"{\"value\":\"first\"}\n{\"value\":\"second\"}";diff --git a/codex-rs/thread-store/src/local/rollout_migration.rs b/codex-rs/thread-store/src/local/rollout_migration.rsindex e08c2461df..facfa952d5 100644--- a/codex-rs/thread-store/src/local/rollout_migration.rs+++ b/codex-rs/thread-store/src/local/rollout_migration.rs@@ -15,8 +15,11 @@ use std::time::Duration;use chrono::DateTime;use codex_app_server_protocol::project_rollout_line;use codex_protocol::ThreadId;+use codex_protocol::protocol::InternalSessionSource;use codex_protocol::protocol::RolloutItem;use codex_protocol::protocol::RolloutLine;+use codex_protocol::protocol::SessionSource;+use codex_protocol::protocol::SubAgentSource;use codex_protocol::protocol::ThreadHistoryMode;use codex_protocol::protocol::ThreadSource;use serde::Serialize;@@ -39,6 +42,10 @@ mod canonicalizer;mod legacy_event;mod line_parser;mod publish;+mod rollback;+mod rollback_plan;+mod rollback_replay;+mod subagent;use canonicalizer::LegacyRolloutCanonicalizer;use publish::compress_rollout_to_path;@@ -48,13 +55,25 @@ use publish::decompressed_staged_rollout_path;use publish::migration_journal_path;use publish::pending_migration_thread_ids;use publish::remove_file_if_present;+use publish::rewrite_subagent_history_boundary;+use publish::rewritten_staged_rollout_path;use publish::staged_rollout_path;use publish::sync_parent_directory;use publish::write_migration_journal;+use rollback_plan::RollbackPlan;+use rollback_plan::RollbackPlanner;const PROJECTION_BATCH_BYTES: u64 = 256 * 1024;const MAX_ROLLOUT_LINE_BYTES: usize = 16 * 1024 * 1024;+enum CanonicalizationAttempt {+ Complete {+ expected_length: u64,+ expected_ordinal: u64,+ },+ NeedsRollbackPlan,+}+struct CanonicalizationSource<'a> {thread_id: ThreadId,source_path: &'a Path,@@ -99,7 +118,7 @@ pub enum RolloutMigrationStatus {Eligible,Migrated,AlreadyPaginated,- SkippedSubagent,+ SkippedEmpty,SkippedBusy,Failed,}@@ -132,6 +151,12 @@ struct RolloutRecord {byte_count: u64,}+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+enum RolloutMigrationKind {+ Ordinary,+ Subagent,+}+impl RolloutMigrationRateLimiter {fn new(max_mib_per_second: u64) -> ThreadStoreResult<Self> {let bytes_per_second = max_mib_per_second@@ -166,7 +191,7 @@ impl RolloutMigrationRateLimiter {}impl LocalThreadStore {- /// Inspect or migrate eligible legacy rollout files beneath active sessions.+ /// Inspect or migrate eligible legacy rollout files beneath active and archived sessions.pub async fn migrate_rollouts(&self,options: RolloutMigrationOptions,@@ -186,6 +211,15 @@ impl LocalThreadStore {let mut paths =find_rollout_paths(&self.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR)).await?;+ paths.extend(+ find_rollout_paths(+ &self+ .config+ .codex_home+ .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR),+ )+ .await?,+ );if options.mode == RolloutMigrationMode::Apply {let pending_thread_ids = pending_migration_thread_ids(&self.config.codex_home).await?;paths.sort_by_key(|path| {@@ -196,138 +230,159 @@ impl LocalThreadStore {let mut report = RolloutMigrationReport::default();for path in paths {- let metadata = match codex_rollout::read_session_meta_line(&path).await {- Ok(metadata) => metadata,- Err(error) => {- let thread_id = thread_id_from_rollout_filename(&path);- if matches_selection(&options.thread_ids, thread_id) {- report.outcomes.push(RolloutMigrationOutcome {- thread_id,- rollout_path: path,- status: RolloutMigrationStatus::Failed,- bytes_processed: 0,- message: Some(error.to_string()),- });- }- continue;- }- };- let thread_id = metadata.meta.id;- if !matches_selection(&options.thread_ids, Some(thread_id)) {- continue;- }- if metadata.meta.source.is_non_root_agent()- || matches!(- metadata.meta.thread_source,- Some(ThreadSource::Subagent | ThreadSource::MemoryConsolidation)- )+ if let Some(outcome) = self+ .migrate_rollout_path(path, &options, &mut limiter)+ .await?{- report.outcomes.push(RolloutMigrationOutcome {- thread_id: Some(thread_id),- rollout_path: path,- status: RolloutMigrationStatus::SkippedSubagent,- bytes_processed: 0,- message: None,- });- continue;+ report.outcomes.push(outcome);}+ }- let journal_path = migration_journal_path(&self.config.codex_home, thread_id);- if metadata.meta.history_mode == ThreadHistoryMode::Paginated {- let bytes_before = limiter.bytes_processed;- let result = if options.mode == RolloutMigrationMode::Apply- && tokio::fs::try_exists(&journal_path)- .await- .map_err(migration_error)?- {- match self- .recover_published_migration(thread_id, &path, &journal_path, &mut limiter)- .await- {- Ok(()) => Ok(RolloutMigrationStatus::Migrated),- Err(ThreadStoreError::Conflict { message }) => {- let bytes_processed =- limiter.bytes_processed.saturating_sub(bytes_before);- report.outcomes.push(skipped_busy_outcome(- thread_id,- path,- message,- bytes_processed,- ));- continue;- }- Err(error) => Err(error),- }- } else {- Ok(RolloutMigrationStatus::AlreadyPaginated)- };- let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);- report- .outcomes- .push(migration_outcome(thread_id, path, result, bytes_processed));- continue;- }+ Ok(report)+ }- if options.mode == RolloutMigrationMode::DryRun {- report.outcomes.push(RolloutMigrationOutcome {- thread_id: Some(thread_id),+ async fn migrate_rollout_path(+ &self,+ mut path: PathBuf,+ options: &RolloutMigrationOptions,+ limiter: &mut RolloutMigrationRateLimiter,+ ) -> ThreadStoreResult<Option<RolloutMigrationOutcome>> {+ let metadata = match codex_rollout::read_session_meta_line(&path).await {+ Ok(metadata) => metadata,+ Err(error) => {+ let thread_id = thread_id_from_rollout_filename(&path);+ if !matches_selection(&options.thread_ids, thread_id) {+ return Ok(None);+ }+ let empty = tokio::fs::metadata(&path)+ .await+ .is_ok_and(|metadata| metadata.len() == 0);+ return Ok(Some(RolloutMigrationOutcome {+ thread_id,rollout_path: path,- status: RolloutMigrationStatus::Eligible,+ status: if empty {+ RolloutMigrationStatus::SkippedEmpty+ } else {+ RolloutMigrationStatus::Failed+ },bytes_processed: 0,- message: None,- });- continue;+ message: (!empty).then(|| error.to_string()),+ }));}+ };+ let thread_id = metadata.meta.id;+ if !matches_selection(&options.thread_ids, Some(thread_id)) {+ return Ok(None);+ }+ let is_memory_consolidation = matches!(+ metadata.meta.source,+ SessionSource::Internal(InternalSessionSource::MemoryConsolidation)+ | SessionSource::SubAgent(SubAgentSource::MemoryConsolidation)+ ) || matches!(+ metadata.meta.thread_source,+ Some(ThreadSource::MemoryConsolidation)+ );+ let kind = if !is_memory_consolidation+ && (matches!(metadata.meta.source, SessionSource::SubAgent(_))+ || matches!(metadata.meta.thread_source, Some(ThreadSource::Subagent)))+ {+ RolloutMigrationKind::Subagent+ } else {+ RolloutMigrationKind::Ordinary+ };- let writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {- Ok(guard) => guard,- Err(ThreadStoreError::Conflict { message }) => {- report.outcomes.push(skipped_busy_outcome(- thread_id, path, message, /*bytes_processed*/ 0,- ));- continue;- }- Err(error) => {- report.outcomes.push(migration_outcome(- thread_id,- path,- Err(error),- /*bytes_processed*/ 0,- ));- continue;- }- };- let bytes_before = limiter.bytes_processed;- let result = match self- .migrate_one_rollout(thread_id, &path, &journal_path, &mut limiter)+ let journal_path = migration_journal_path(&self.config.codex_home, thread_id);+ let pending_published_migration = metadata.meta.history_mode+ == ThreadHistoryMode::Paginated+ && options.mode == RolloutMigrationMode::Apply+ && tokio::fs::try_exists(&journal_path).await- {- Ok(()) => Ok(RolloutMigrationStatus::Migrated),- Err(error) => {- if let Err(cleanup_error) = self- .cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)- .await- {- Err(migration_error(format!(- "{error}; failed to clean up unpublished migration: {cleanup_error}"- )))- } else {- Err(error)+ .map_err(migration_error)?;+ if metadata.meta.history_mode == ThreadHistoryMode::Paginated {+ let bytes_before = limiter.bytes_processed;+ let result = if pending_published_migration {+ match self+ .recover_published_migration(thread_id, &path, &journal_path, limiter)+ .await+ {+ Ok(recovered_path) => {+ path = recovered_path;+ Ok(RolloutMigrationStatus::Migrated)+ }+ Err(ThreadStoreError::Conflict { message }) => {+ let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);+ return Ok(Some(skipped_busy_outcome(+ thread_id,+ path,+ message,+ bytes_processed,+ )));}+ Err(error) => Err(error),}+ } else {+ Ok(RolloutMigrationStatus::AlreadyPaginated)};- drop(writer_guard);let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);- let outcome = match result {- Err(ThreadStoreError::Conflict { message }) => {- skipped_busy_outcome(thread_id, path, message, bytes_processed)- }- result => migration_outcome(thread_id, path, result, bytes_processed),- };- report.outcomes.push(outcome);+ return Ok(Some(migration_outcome(+ thread_id,+ path,+ result,+ bytes_processed,+ )));}- Ok(report)+ if options.mode == RolloutMigrationMode::DryRun {+ return Ok(Some(migration_outcome(+ thread_id,+ path,+ Ok(RolloutMigrationStatus::Eligible),+ /*bytes_processed*/ 0,+ )));+ }++ let _writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {+ Ok(guard) => guard,+ Err(ThreadStoreError::Conflict { message }) => {+ return Ok(Some(skipped_busy_outcome(+ thread_id, path, message, /*bytes_processed*/ 0,+ )));+ }+ Err(error) => {+ return Ok(Some(migration_outcome(+ thread_id,+ path,+ Err(error),+ /*bytes_processed*/ 0,+ )));+ }+ };+ let bytes_before = limiter.bytes_processed;+ let result = match self+ .migrate_one_rollout(thread_id, &path, &journal_path, kind, limiter)+ .await+ {+ Ok(()) => Ok(RolloutMigrationStatus::Migrated),+ Err(error) => {+ if let Err(cleanup_error) = self+ .cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)+ .await+ {+ Err(migration_error(format!(+ "{error}; failed to clean up unpublished migration: {cleanup_error}"+ )))+ } else {+ Err(error)+ }+ }+ };+ let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);+ Ok(Some(match result {+ Err(ThreadStoreError::Conflict { message }) => {+ skipped_busy_outcome(thread_id, path, message, bytes_processed)+ }+ result => migration_outcome(thread_id, path, result, bytes_processed),+ }))}async fn migrate_one_rollout(@@ -335,6 +390,7 @@ impl LocalThreadStore {thread_id: ThreadId,rollout_path: &Path,journal_path: &Path,+ kind: RolloutMigrationKind,limiter: &mut RolloutMigrationRateLimiter,) -> ThreadStoreResult<()> {if let Some(state_db) = &self.state_db@@ -403,8 +459,31 @@ impl LocalThreadStore {source_permissions: &source_permissions,canonical_session_meta: &canonical_session_meta,};- let (expected_length, expected_ordinal) =- Self::write_canonical_rollout(&canonicalization_source, limiter).await?;+ let bounded_subagent_context = if kind == RolloutMigrationKind::Subagent {+ let RolloutItem::SessionMeta(session_meta) = &canonical_session_meta.item else {+ return Err(migration_error("canonical session metadata is missing"));+ };+ let context =+ subagent::select_bounded_context(source_path.to_path_buf(), session_meta.clone())+ .await?;+ limiter.account(source_metadata.len()).await;+ context+ } else {+ None+ };+ let (_, expected_ordinal) = if let Some(items) = bounded_subagent_context {+ Self::write_bounded_subagent_rollout(&canonicalization_source, items, limiter).await?+ } else {+ Self::write_rollout_with_rollback_plan(&canonicalization_source, limiter).await?+ };++ if kind == RolloutMigrationKind::Subagent {+ rewrite_subagent_history_boundary(&staged_path, expected_ordinal).await?;+ }+ let expected_length = tokio::fs::metadata(&staged_path)+ .await+ .map_err(migration_error)?+ .len();// SQLite projection only starts after every staged-file mutation is durable.let modified_at = source_modified;@@ -473,10 +552,97 @@ impl LocalThreadStore {.await}- async fn write_canonical_rollout(+ async fn build_rollback_plan(+ source_path: &Path,+ limiter: &mut RolloutMigrationRateLimiter,+ ) -> ThreadStoreResult<RollbackPlan> {+ let source_file = File::open(source_path).await.map_err(migration_error)?;+ let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);+ let mut bytes = Vec::new();+ let mut planner = RollbackPlanner::new();+ while let Some(record) = read_rollout_record(&mut source, &mut bytes).await? {+ limiter.account(record.byte_count).await;+ if let Some(line) = record.line {+ planner.observe(&line)?;+ }+ }+ Ok(planner.finish())+ }++ async fn write_rollout_with_rollback_plan(+ input: &CanonicalizationSource<'_>,+ limiter: &mut RolloutMigrationRateLimiter,+ ) -> ThreadStoreResult<(u64, u64)> {+ let attempt = Self::write_canonical_rollout(input, /*plan*/ None, limiter).await?;+ match attempt {+ CanonicalizationAttempt::Complete {+ expected_length,+ expected_ordinal,+ } => Ok((expected_length, expected_ordinal)),+ CanonicalizationAttempt::NeedsRollbackPlan => {+ remove_file_if_present(input.staged_path).await?;+ let plan = Self::build_rollback_plan(input.source_path, limiter).await?;+ let CanonicalizationAttempt::Complete {+ expected_length,+ expected_ordinal,+ } = Self::write_canonical_rollout(input, Some(&plan), limiter).await?+ else {+ return Err(migration_error(+ "planned rollout still contains a rollback marker",+ ));+ };+ Ok((expected_length, expected_ordinal))+ }+ }+ }++ async fn write_bounded_subagent_rollout(input: &CanonicalizationSource<'_>,+ items: Vec<RolloutItem>,limiter: &mut RolloutMigrationRateLimiter,) -> ThreadStoreResult<(u64, u64)> {+ let staged_file = tokio::fs::OpenOptions::new()+ .create(true)+ .truncate(true)+ .write(true)+ .open(input.staged_path)+ .await+ .map_err(migration_error)?;+ staged_file+ .set_permissions(input.source_permissions.clone())+ .await+ .map_err(migration_error)?;+ let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);+ let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);+ let written = canonicalizer+ .write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)+ .await?;+ limiter.account(written).await;+ for item in items {+ let line = RolloutLine {+ timestamp: input.canonical_session_meta.timestamp.clone(),+ ordinal: None,+ item,+ };+ let written = canonicalizer.process_line(line, &mut staged).await?;+ limiter.account(written).await;+ }+ let written = canonicalizer+ .finish(&mut staged, &input.canonical_session_meta.timestamp)+ .await?;+ limiter.account(written).await;+ staged.flush().await.map_err(migration_error)?;+ Ok((+ canonicalizer.output_byte_offset(),+ canonicalizer.next_ordinal(),+ ))+ }++ async fn write_canonical_rollout(+ input: &CanonicalizationSource<'_>,+ plan: Option<&RollbackPlan>,+ limiter: &mut RolloutMigrationRateLimiter,+ ) -> ThreadStoreResult<CanonicalizationAttempt> {let source_file = File::open(input.source_path).await.map_err(migration_error)?;@@ -494,6 +660,7 @@ impl LocalThreadStore {let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);let mut bytes = Vec::new();+ let mut parsed_record_index = 0_usize;let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);let written = canonicalizer.write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)@@ -506,24 +673,45 @@ impl LocalThreadStore {let Some(line) = record.line else {continue;};- if matches!(- &line.item,- RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))- ) {- return Err(migration_error("legacy rollout contains a rollback marker"));+ if plan.is_none()+ && matches!(+ &line.item,+ RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))+ )+ {+ return Ok(CanonicalizationAttempt::NeedsRollbackPlan);}+ let line = if let Some(plan) = plan {+ let planned = plan.apply(parsed_record_index, line)?;+ parsed_record_index = parsed_record_index+ .checked_add(1)+ .ok_or_else(|| migration_error("legacy rollout record index overflow"))?;+ let Some(line) = planned else {+ continue;+ };+ line+ } else {+ line+ };last_timestamp = line.timestamp.clone();let written = canonicalizer.process_line(line, &mut staged).await?;limiter.account(written).await;}+ if let Some(plan) = plan+ && parsed_record_index != plan.record_count()+ {+ return Err(migration_error(+ "rollback plan source length changed during replay",+ ));+ }let written = canonicalizer.finish(&mut staged, &last_timestamp).await?;limiter.account(written).await;staged.flush().await.map_err(migration_error)?;- Ok((- canonicalizer.output_byte_offset(),- canonicalizer.next_ordinal(),- ))+ Ok(CanonicalizationAttempt::Complete {+ expected_length: canonicalizer.output_byte_offset(),+ expected_ordinal: canonicalizer.next_ordinal(),+ })}async fn recover_published_migration(@@ -532,7 +720,7 @@ impl LocalThreadStore {rollout_path: &Path,journal_path: &Path,limiter: &mut RolloutMigrationRateLimiter,- ) -> ThreadStoreResult<()> {+ ) -> ThreadStoreResult<PathBuf> {let _writer_guard = self.writer_lock_coordinator.acquire(thread_id)?;let decompressed_path = rollout_path_is_compressed(rollout_path).then(|| decompressed_staged_rollout_path(rollout_path))@@ -572,7 +760,8 @@ impl LocalThreadStore {remove_file_if_present(decompressed_path).await?;}self.finish_published_migration(thread_id, journal_path)- .await+ .await?;+ Ok(rollout_path.to_path_buf())}async fn cleanup_failed_unpublished_migration(@@ -589,7 +778,9 @@ impl LocalThreadStore {}thread_history::delete_thread(self, thread_id).await?;- remove_file_if_present(&staged_rollout_path(rollout_path)?).await?;+ let staged_path = staged_rollout_path(rollout_path)?;+ remove_file_if_present(&staged_path).await?;+ remove_file_if_present(&rewritten_staged_rollout_path(&staged_path)?).await?;remove_file_if_present(&compressed_staged_rollout_path(rollout_path)?).await?;remove_file_if_present(&decompressed_staged_rollout_path(rollout_path)?).await?;remove_file_if_present(journal_path).await?;@@ -623,6 +814,11 @@ impl LocalThreadStore {rollout_path: &Path,limiter: &mut RolloutMigrationRateLimiter,) -> ThreadStoreResult<()> {+ let subagent_history_start_ordinal = codex_rollout::read_session_meta_line(rollout_path)+ .await+ .map_err(migration_error)?+ .meta+ .subagent_history_start_ordinal;let file = File::open(rollout_path).await.map_err(migration_error)?;let mut reader = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, file);let mut line_bytes = Vec::new();@@ -650,7 +846,11 @@ impl LocalThreadStore {start_byte_offset: offset,end_byte_offset: next_offset,fallback_created_at_ms: Some(fallback_created_at_ms),- changes: project_rollout_line(&line),+ changes: if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) {+ Default::default()+ } else {+ project_rollout_line(&line)+ },}));offset = next_offset;@@ -688,7 +888,7 @@ async fn read_rollout_record(bytes: &mut Vec<u8>,) -> ThreadStoreResult<Option<RolloutRecord>> {bytes.clear();- let byte_count = reader+ let mut byte_count = reader.take((MAX_ROLLOUT_LINE_BYTES + 1) as u64).read_until(b'\n', bytes).await@@ -697,11 +897,30 @@ async fn read_rollout_record(return Ok(None);}if byte_count > MAX_ROLLOUT_LINE_BYTES {- return Err(migration_error(- "rollout contains an oversized JSONL record",- ));+ // Some historical tool outputs are too large to migrate safely in memory. Discard the+ // whole record, including any unread suffix.+ while bytes.last() != Some(&b'\n') {+ bytes.clear();+ let chunk_bytes = reader+ .take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)+ .read_until(b'\n', bytes)+ .await+ .map_err(migration_error)?;+ if chunk_bytes == 0 {+ break;+ }+ byte_count = byte_count+ .checked_add(chunk_bytes)+ .ok_or_else(|| migration_error("rollout record byte count overflow"))?;+ }+ return Ok(Some(RolloutRecord {+ line: None,+ byte_count: byte_count as u64,+ }));}- let line = line_parser::parse_legacy_rollout_line(bytes).map_err(migration_error)?;+ // Legacy records do not have ordinals, so malformed complete records cannot be repaired.+ // Skip them and let the next newline-delimited record resynchronize the stream.+ let line = line_parser::parse_legacy_rollout_line(bytes).unwrap_or(None);Ok(Some(RolloutRecord {line,byte_count: byte_count as u64,@@ -734,7 +953,15 @@ async fn find_rollout_paths(root: &Path) -> ThreadStoreResult<Vec<PathBuf>> {if name.starts_with("rollout-")&& (name.ends_with(".jsonl") || name.ends_with(".jsonl.zst")){- paths.push(entry.path());+ let path = entry.path();+ if name.ends_with(".jsonl.zst")+ && tokio::fs::try_exists(codex_rollout::plain_rollout_path(&path))+ .await+ .map_err(migration_error)?+ {+ continue;+ }+ paths.push(path);}}}@@ -747,6 +974,12 @@ fn matches_selection(selected: &[ThreadId], actual: Option<ThreadId>) -> bool {selected.is_empty() || actual.is_some_and(|thread_id| selected.contains(&thread_id))}+fn rollout_path_is_compressed(path: &Path) -> bool {+ path.file_name()+ .and_then(|name| name.to_str())+ .is_some_and(|name| name.ends_with(".jsonl.zst"))+}+fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {let name = path.file_name()?.to_str()?;let stem = name@@ -756,12 +989,6 @@ fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {ThreadId::from_string(stem.get(start..)?).ok()}-fn rollout_path_is_compressed(path: &Path) -> bool {- path.file_name()- .and_then(|name| name.to_str())- .is_some_and(|name| name.ends_with(".jsonl.zst"))-}-fn migration_outcome(thread_id: ThreadId,rollout_path: PathBuf,diff --git a/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs b/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rsindex d1519fe697..87ce548d5d 100644--- a/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs+++ b/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs@@ -1,8 +1,9 @@//! Replays normalized legacy rollout records into canonical paginated JSONL.//!//! `line_parser` makes old JSON shapes parseable, `legacy_event` converts obsolete completion-//! events into modern turn items. This module assigns stable ordinals, keeps `SessionMeta` at-//! ordinal zero, and emits the surviving history selected by the caller.+//! events into modern turn items. Rollback planning happens before this writer sees a record, so+//! this module only assigns stable ordinals, keeps `SessionMeta` at ordinal zero, and emits the+//! already-selected surviving history.//!//! The goal is to preserve the model-visible conversation, not to preserve every legacy record//! byte-for-byte. Filesystem publishing and SQLite projection intentionally live outside this@@ -52,7 +53,6 @@ pub(super) struct LegacyRolloutCanonicalizer {active_turn: Option<ActiveTurn>,known_turn_ids: HashSet<String>,reasoning: Option<ReasoningItem>,- saw_source_session_meta: bool,}impl LegacyRolloutCanonicalizer {@@ -67,18 +67,17 @@ impl LegacyRolloutCanonicalizer {active_turn: None,known_turn_ids: HashSet::new(),reasoning: None,- saw_source_session_meta: false,}}- pub(super) fn output_byte_offset(&self) -> u64 {- self.output_byte_offset- }-pub(super) fn next_ordinal(&self) -> u64 {self.next_ordinal}+ pub(super) fn output_byte_offset(&self) -> u64 {+ self.output_byte_offset+ }+pub(super) async fn write_head_session_meta(&mut self,line: RolloutLine,@@ -114,14 +113,7 @@ impl LegacyRolloutCanonicalizer {let timestamp = line.timestamp;let bytes_before = self.bytes_written;match line.item {- RolloutItem::SessionMeta(metadata) => {- if !self.saw_source_session_meta {- self.saw_source_session_meta = true;- return Ok(0);- }- self.write_item(writer, ×tamp, RolloutItem::SessionMeta(metadata))- .await?;- }+ RolloutItem::SessionMeta(_) => return Ok(0),RolloutItem::ResponseItem(ResponseItem::Other) => {return Err(migration_error("legacy rollout contains an unsupported response item",diff --git a/codex-rs/thread-store/src/local/rollout_migration/line_parser.rs b/codex-rs/thread-store/src/local/rollout_migration/line_parser.rsindex 872e2a3e7e..dd71bcd8e5 100644--- a/codex-rs/thread-store/src/local/rollout_migration/line_parser.rs+++ b/codex-rs/thread-store/src/local/rollout_migration/line_parser.rs@@ -26,7 +26,11 @@ pub(super) fn parse_legacy_rollout_line(bytes: &[u8]) -> Result<Option<RolloutLi// Deserializing through Value is intentional. Some historical numeric// payloads fail serde_json's streaming enum path but deserialize correctly// once their tagged object shape has been materialized.- let mut value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;+ let value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;+ parse_legacy_rollout_value(value)+}++pub(super) fn parse_legacy_rollout_value(mut value: Value) -> Result<Option<RolloutLine>, String> {if should_skip_retired_record(&value) {return Ok(None);}diff --git a/codex-rs/thread-store/src/local/rollout_migration/publish.rs b/codex-rs/thread-store/src/local/rollout_migration/publish.rsindex 21f142e3c2..cd42249970 100644--- a/codex-rs/thread-store/src/local/rollout_migration/publish.rs+++ b/codex-rs/thread-store/src/local/rollout_migration/publish.rs@@ -22,6 +22,13 @@ use std::os::unix::fs::OpenOptionsExt;use std::os::unix::fs::PermissionsExt;use codex_protocol::ThreadId;+use codex_protocol::protocol::RolloutItem;+use codex_protocol::protocol::RolloutLine;+use tokio::fs::File;+use tokio::io::AsyncBufReadExt;+use tokio::io::AsyncWriteExt;+use tokio::io::BufReader;+use tokio::io::BufWriter;use super::migration_error;use crate::ThreadStoreResult;@@ -139,6 +146,74 @@ pub(super) async fn compress_rollout_to_path(.map_err(migration_error)}+pub(super) fn rewritten_staged_rollout_path(staged_path: &Path) -> ThreadStoreResult<PathBuf> {+ let filename = staged_path+ .file_name()+ .and_then(|filename| filename.to_str())+ .ok_or_else(|| migration_error("staged rollout path has no valid filename"))?;+ Ok(staged_path.with_file_name(format!("{filename}.head.tmp")))+}++pub(super) async fn rewrite_subagent_history_boundary(+ staged_path: &Path,+ boundary: u64,+) -> ThreadStoreResult<()> {+ let rewritten_path = rewritten_staged_rollout_path(staged_path)?;+ let permissions = tokio::fs::metadata(staged_path)+ .await+ .map_err(migration_error)?+ .permissions();+ let source = File::open(staged_path).await.map_err(migration_error)?;+ let mut source = BufReader::new(source);+ let mut head_bytes = Vec::new();+ source+ .read_until(b'\n', &mut head_bytes)+ .await+ .map_err(migration_error)?;+ let mut head = serde_json::from_slice::<RolloutLine>(&head_bytes).map_err(migration_error)?;+ let RolloutItem::SessionMeta(session_meta) = &mut head.item else {+ return Err(migration_error(+ "staged rollout head is not session metadata",+ ));+ };+ session_meta.meta.subagent_history_start_ordinal = Some(boundary);++ let rewritten = tokio::fs::OpenOptions::new()+ .create(true)+ .truncate(true)+ .write(true)+ .open(&rewritten_path)+ .await+ .map_err(migration_error)?;+ rewritten+ .set_permissions(permissions)+ .await+ .map_err(migration_error)?;+ let mut rewritten = BufWriter::new(rewritten);+ rewritten+ .write_all(+ serde_json::to_string(&head)+ .map_err(migration_error)?+ .as_bytes(),+ )+ .await+ .map_err(migration_error)?;+ rewritten.write_all(b"\n").await.map_err(migration_error)?;+ tokio::io::copy(&mut source, &mut rewritten)+ .await+ .map_err(migration_error)?;+ rewritten.flush().await.map_err(migration_error)?;+ rewritten+ .get_ref()+ .sync_all()+ .await+ .map_err(migration_error)?;+ drop(rewritten);+ tokio::fs::rename(rewritten_path, staged_path)+ .await+ .map_err(migration_error)+}+pub(super) async fn remove_file_if_present(path: &Path) -> ThreadStoreResult<()> {if tokio::fs::try_exists(path).await.map_err(migration_error)? {tokio::fs::remove_file(path)diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback.rsnew file mode 100644index 0000000000..428aecb45b--- /dev/null+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback.rs@@ -0,0 +1,140 @@+//! Defines which persisted history entries count as rollbackable user turns.+//!+//! Legacy `ThreadRolledBack { num_turns }` does not mean “drop the last N JSONL records.” A turn+//! can include contextual user/developer messages, paired `ResponseItem` + `UserMessage` records,+//! inter-agent communication, or compaction replacement history.+//!+//! This module mirrors the legacy cold-resume rollback boundary rules for persisted rollout shapes.+//! It stays local to migration because core's live predicate also knows about dynamically+//! registered contextual fragments that thread-store should not depend on.++use codex_protocol::items::parse_hook_prompt_fragment;+use codex_protocol::models::ContentItem;+use codex_protocol::models::ResponseItem;+use codex_protocol::protocol::InterAgentCommunication;++/// Match the rollback boundaries used by legacy history reconstruction without+/// treating every persisted lifecycle as a user turn. This intentionally stays+/// local to the frozen migration adapter: the full core predicate also knows+/// about dynamically registered contextual fragments that thread-store cannot+/// depend on.+pub(super) fn counts_as_boundary(response: &ResponseItem) -> bool {+ if matches!(response, ResponseItem::AgentMessage { .. }) {+ return true;+ }+ let ResponseItem::Message { role, content, .. } = response else {+ return false;+ };+ (role == "user" && !is_known_contextual_user_message_content(content))+ || (role == "assistant" && InterAgentCommunication::is_message_content(content))+}++pub(super) fn is_pre_turn_context_update(response: &ResponseItem) -> bool {+ let ResponseItem::Message { role, content, .. } = response else {+ return false;+ };+ (role == "user" && is_known_contextual_user_message_content(content))+ || (role == "developer" && is_known_contextual_developer_message_content(content))+}++/// Apply the same response-history cut used by legacy cold resume to a persisted+/// compaction checkpoint. The migration adapter uses a frozen contextual-fragment+/// matcher because thread-store cannot depend on core's runtime fragment registry.+pub(super) fn drop_last_n_user_turns(history: &mut Vec<ResponseItem>, num_turns: u32) {+ if num_turns == 0 {+ return;+ }+ let user_positions = history+ .iter()+ .enumerate()+ .filter_map(|(index, item)| counts_as_boundary(item).then_some(index))+ .collect::<Vec<_>>();+ let Some(&first_turn_index) = user_positions.first() else {+ return;+ };+ let count = usize::try_from(num_turns).unwrap_or(usize::MAX);+ let mut cut_index = if count >= user_positions.len() {+ first_turn_index+ } else {+ user_positions[user_positions.len() - count]+ };+ while cut_index > first_turn_index && is_pre_turn_context_update(&history[cut_index - 1]) {+ cut_index -= 1;+ }+ history.truncate(cut_index);+}++fn is_known_contextual_user_message_content(content: &[ContentItem]) -> bool {+ content.iter().any(|item| {+ let ContentItem::InputText { text } = item else {+ return false;+ };+ is_known_contextual_user_text(text)+ })+}++fn is_known_contextual_developer_message_content(content: &[ContentItem]) -> bool {+ content.iter().any(|item| {+ let ContentItem::InputText { text } = item else {+ return false;+ };+ let text = text.trim_start();+ [+ "<permissions instructions>",+ "<model_switch>",+ "<apps_instructions>",+ "<collaboration_mode>",+ "<multi_agent_mode>",+ "<environments_instructions>",+ "<git_attribution>",+ "<plugins_instructions>",+ "<realtime_conversation>",+ "<skills_instructions>",+ "<tools>",+ "<personality_spec>",+ "<token_budget>",+ "<context_window>",+ "<context_window_guidance>",+ "<rollout_budget>",+ ]+ .iter()+ .any(|prefix| {+ text.get(..prefix.len())+ .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))+ })+ })+}++// Keep this frozen alongside the legacy migration adapter. Core's live predicate also knows+// about dynamically registered fragments, but legacy rollouts only need these persisted shapes.+fn is_known_contextual_user_text(text: &str) -> bool {+ let text = text.trim();+ parse_hook_prompt_fragment(text).is_some()+ || [+ ("# AGENTS.md instructions", "</INSTRUCTIONS>"),+ ("<environment_context>", "</environment_context>"),+ ("<skill>", "</skill>"),+ ("<user_shell_command>", "</user_shell_command>"),+ ("<turn_aborted>", "</turn_aborted>"),+ ("<subagent_notification>", "</subagent_notification>"),+ ("<recommended_plugins>", "</recommended_plugins>"),+ ("<goal_context>", "</goal_context>"),+ ]+ .iter()+ .any(|(start, end)| text.starts_with(start) && text.ends_with(end))+ || (text.starts_with("<external_")+ && text+ .split_once('>')+ .and_then(|(start, _)| start.strip_prefix("<external_"))+ .is_some_and(|key| text.ends_with(&format!("</external_{key}>"))))+ || (text.starts_with("<codex_internal_context")+ && text.ends_with("</codex_internal_context>"))+ || text.starts_with(+ "Warning: The maximum number of unified exec processes you can keep open is",+ )+ || (text.starts_with("Warning: apply_patch was requested via ")+ && text.ends_with("Use the apply_patch tool instead of exec_command."))+ || text.starts_with(+ "Warning: Your account was flagged for potentially high-risk cyber activity",+ )+}diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rsnew file mode 100644index 0000000000..d28c1c09a2--- /dev/null+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs@@ -0,0 +1,380 @@+//! Decides which legacy records remain visible after historical rollback.+//!+//! Legacy rollback removes logical instruction turns, not a physical suffix of the rollout file.+//! Most records happen to be ordered that way, but late completion events can target an older+//! surviving turn after a newer turn has started. This planner keeps compact per-record ownership+//! metadata for SQLite visibility, then combines it with `rollback_replay`'s cold-resume answer+//! before the writer makes its second streaming pass.++use std::collections::HashMap;++use codex_protocol::models::ContentItem;+use codex_protocol::models::ResponseItem;+use codex_protocol::protocol::CompactedItem;+use codex_protocol::protocol::EventMsg;+use codex_protocol::protocol::RolloutItem;+use codex_protocol::protocol::RolloutLine;+use codex_protocol::protocol::UserMessageEvent;++use super::migration_error;+use super::rollback;+use super::rollback_replay::ModelReplayPlanner;+use crate::ThreadStoreResult;++#[derive(Clone)]+struct CompactionFrame {+ record_index: usize,+ boundary_depth: usize,+ owner: Option<usize>,+ item: CompactedItem,+}++#[derive(Clone)]+struct PendingUserResponse {+ boundary: usize,+ content: Vec<ContentItem>,+}++/// Compact plan keyed by parsed source-record index.+pub(super) struct RollbackPlan {+ record_boundaries: Vec<Option<usize>>,+ boundary_alive: Vec<bool>,+ compacted_items: HashMap<usize, CompactedItem>,+}++impl RollbackPlan {+ pub(super) fn record_count(&self) -> usize {+ self.record_boundaries.len()+ }++ pub(super) fn apply(+ &self,+ record_index: usize,+ mut line: RolloutLine,+ ) -> ThreadStoreResult<Option<RolloutLine>> {+ let boundary = self+ .record_boundaries+ .get(record_index)+ .ok_or_else(|| migration_error("rollback plan is shorter than source replay"))?;+ if matches!(+ &line.item,+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))+ ) {+ return Ok(None);+ }+ // A rolled-back turn can still own the empty checkpoint that keeps cold resume from+ // replaying older history.+ if let Some(compacted) = self.compacted_items.get(&record_index) {+ line.item = RolloutItem::Compacted(compacted.clone());+ return Ok(Some(line));+ }+ if boundary.is_some_and(|boundary| !self.boundary_alive[boundary]) {+ return Ok(None);+ }+ Ok(Some(line))+ }+}++/// Streaming builder for RollbackPlan.+pub(super) struct RollbackPlanner {+ record_boundaries: Vec<Option<usize>>,+ boundary_alive: Vec<bool>,+ boundary_stack: Vec<usize>,+ active_turn_id: Option<String>,+ pending_turn_records: Vec<usize>,+ pending_context_records: Vec<usize>,+ pending_user_response: Option<PendingUserResponse>,+ pending_delivery_boundary: Option<usize>,+ turn_boundaries: HashMap<String, usize>,+ compactions: Vec<CompactionFrame>,+ model_replay: ModelReplayPlanner,+}++impl RollbackPlanner {+ pub(super) fn new() -> Self {+ Self {+ record_boundaries: Vec::new(),+ boundary_alive: Vec::new(),+ boundary_stack: Vec::new(),+ active_turn_id: None,+ pending_turn_records: Vec::new(),+ pending_context_records: Vec::new(),+ pending_user_response: None,+ pending_delivery_boundary: None,+ turn_boundaries: HashMap::new(),+ compactions: Vec::new(),+ model_replay: ModelReplayPlanner::new(),+ }+ }++ pub(super) fn observe(&mut self, line: &RolloutLine) -> ThreadStoreResult<()> {+ let index = self.record_boundaries.len();+ self.model_replay.observe(index, &line.item);+ self.record_boundaries+ .push(self.boundary_stack.last().copied());+ let paired_user_boundary = match (&self.pending_user_response, &line.item) {+ (Some(pending), RolloutItem::EventMsg(EventMsg::UserMessage(event)))+ if user_response_matches_event(&pending.content, event) =>+ {+ Some(pending.boundary)+ }+ _ => None,+ };+ let paired_delivery_boundary = match (&self.pending_delivery_boundary, &line.item) {+ (Some(boundary), RolloutItem::ResponseItem(ResponseItem::AgentMessage { .. })) => {+ Some(*boundary)+ }+ _ => None,+ };+ self.pending_user_response = None;+ self.pending_delivery_boundary = None;++ match &line.item {+ RolloutItem::SessionMeta(_) => self.record_boundaries[index] = None,+ RolloutItem::ResponseItem(response) => {+ if let Some(boundary) = paired_delivery_boundary {+ self.record_boundaries[index] = Some(boundary);+ } else if rollback::counts_as_boundary(response) {+ let boundary = self.start_boundary(index);+ if let ResponseItem::Message { role, content, .. } = response+ && role == "user"+ {+ self.pending_user_response = Some(PendingUserResponse {+ boundary,+ content: content.clone(),+ });+ }+ } else if rollback::is_pre_turn_context_update(response) {+ // Until another user boundary arrives, this is trailing context for the+ // previous turn. Keep that fallback owner so rollback drops it when there is+ // no later turn to attach it to.+ self.pending_context_records.push(index);+ }+ }+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {+ self.apply_rollback(rollback.num_turns)?;+ }+ RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {+ self.active_turn_id = Some(event.turn_id.clone());+ self.pending_turn_records.clear();+ self.pending_turn_records.push(index);+ }+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {+ self.assign_targeted_record(index, Some(event.turn_id.as_str()));+ if self.active_turn_id.as_deref() == Some(event.turn_id.as_str()) {+ self.active_turn_id = None;+ self.pending_turn_records.clear();+ }+ }+ RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {+ self.assign_targeted_record(index, event.turn_id.as_deref());+ if event+ .turn_id+ .as_deref()+ .is_some_and(|turn_id| self.active_turn_id.as_deref() == Some(turn_id))+ {+ self.active_turn_id = None;+ self.pending_turn_records.clear();+ }+ }+ RolloutItem::EventMsg(EventMsg::UserMessage(_)) => {+ let boundary = paired_user_boundary.unwrap_or_else(|| self.start_boundary(index));+ self.record_boundaries[index] = Some(boundary);+ }+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => {+ self.assign_targeted_record(index, Some(event.turn_id.as_str()));+ }+ RolloutItem::EventMsg(event) => {+ self.assign_targeted_record(index, explicit_event_turn_id(event));+ }+ RolloutItem::InterAgentCommunication(_) => {+ self.start_boundary(index);+ }+ RolloutItem::InterAgentCommunicationMetadata { .. } => {+ let boundary = self.start_boundary(index);+ self.pending_delivery_boundary = Some(boundary);+ }+ RolloutItem::Compacted(item) => {+ let owner = self+ .active_turn_id+ .as_deref()+ .and_then(|turn_id| self.turn_boundaries.get(turn_id).copied());+ self.record_boundaries[index] = owner;+ self.compactions.push(CompactionFrame {+ record_index: index,+ boundary_depth: self.boundary_stack.len(),+ owner,+ item: item.clone(),+ });+ }+ RolloutItem::TurnContext(_) => {+ if self.active_turn_id.is_some()+ && self+ .active_turn_id+ .as_deref()+ .is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))+ {+ self.pending_turn_records.push(index);+ }+ }+ RolloutItem::WorldState(_) => {}+ }++ Ok(())+ }++ pub(super) fn finish(self) -> RollbackPlan {+ let RollbackPlanner {+ record_boundaries,+ boundary_alive,+ compactions,+ model_replay,+ ..+ } = self;+ let replay_anchor = model_replay.finish().empty_replacement_history_compaction;+ let compacted_items = compactions+ .into_iter()+ .filter_map(|mut frame| {+ if Some(frame.record_index) == replay_anchor {+ frame.item.replacement_history = Some(Vec::new());+ return Some((frame.record_index, frame.item));+ }+ frame+ .owner+ .is_none_or(|boundary| boundary_alive[boundary])+ .then_some((frame.record_index, frame.item))+ })+ .collect::<HashMap<_, _>>();+ RollbackPlan {+ record_boundaries,+ boundary_alive,+ compacted_items,+ }+ }++ fn start_boundary(&mut self, index: usize) -> usize {+ let boundary = self.boundary_alive.len();+ self.boundary_alive.push(true);+ let had_prior_boundary = !self.boundary_stack.is_empty();+ if had_prior_boundary {+ for pending_index in self.pending_context_records.drain(..) {+ self.record_boundaries[pending_index] = Some(boundary);+ }+ } else {+ self.pending_context_records.clear();+ }+ for pending_index in self.pending_turn_records.drain(..) {+ self.record_boundaries[pending_index] = Some(boundary);+ }+ self.record_boundaries[index] = Some(boundary);+ self.boundary_stack.push(boundary);+ self.bind_active_turn(boundary);+ boundary+ }++ fn bind_active_turn(&mut self, boundary: usize) {+ if let Some(turn_id) = self.active_turn_id.as_ref() {+ self.turn_boundaries.insert(turn_id.clone(), boundary);+ }+ }++ fn assign_targeted_record(&mut self, index: usize, turn_id: Option<&str>) {+ if let Some(boundary) = turn_id.and_then(|turn_id| self.turn_boundaries.get(turn_id)) {+ self.record_boundaries[index] = Some(*boundary);+ } else if self.active_turn_id.is_some()+ && self+ .active_turn_id+ .as_deref()+ .is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))+ {+ self.pending_turn_records.push(index);+ }+ }++ fn apply_rollback(&mut self, num_turns: u32) -> ThreadStoreResult<()> {+ let count = usize::try_from(num_turns).unwrap_or(usize::MAX);+ if count == 0 {+ return Ok(());+ }+ let depth_before = self.boundary_stack.len();+ for _ in 0..count {+ let Some(boundary) = self.boundary_stack.pop() else {+ break;+ };+ self.boundary_alive[boundary] = false;+ }+ let compaction_index = self.compactions.iter().rposition(|frame| {+ frame+ .owner+ .is_none_or(|boundary| self.boundary_alive[boundary])+ });+ if let Some(compaction_index) = compaction_index {+ let frame = &mut self.compactions[compaction_index];+ let post_compaction_turns = depth_before.saturating_sub(frame.boundary_depth);+ let remaining = count.saturating_sub(post_compaction_turns);+ if remaining > 0 {+ let replacement_history =+ frame.item.replacement_history.as_mut().ok_or_else(|| {+ migration_error(+ "legacy rollback crosses a compaction without replacement history",+ )+ })?;+ rollback::drop_last_n_user_turns(+ replacement_history,+ u32::try_from(remaining).unwrap_or(u32::MAX),+ );+ }+ }+ self.active_turn_id = None;+ self.pending_turn_records.clear();+ self.pending_context_records.clear();+ self.pending_user_response = None;+ self.pending_delivery_boundary = None;+ Ok(())+ }+}++fn explicit_event_turn_id(event: &EventMsg) -> Option<&str> {+ match event {+ EventMsg::ExecCommandEnd(event) => Some(event.turn_id.as_str()),+ EventMsg::PatchApplyEnd(event) => Some(event.turn_id.as_str()),+ EventMsg::DynamicToolCallResponse(event) => Some(event.turn_id.as_str()),+ EventMsg::EnteredReviewMode(event) => event.turn_id.as_deref(),+ EventMsg::ExitedReviewMode(event) => event.turn_id.as_deref(),+ _ => None,+ }+ .filter(|turn_id| !turn_id.is_empty())+}++fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent) -> bool {+ let mut text = String::new();+ let mut images = Vec::new();+ let mut audio = Vec::new();+ for item in content {+ match item {+ ContentItem::InputText { text: item_text } => text.push_str(item_text),+ ContentItem::InputImage { image_url, .. } => images.push(image_url.as_str()),+ ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),+ ContentItem::OutputText { .. } => return false,+ }+ }+ text == event.message+ && images+ == event+ .images+ .as_deref()+ .unwrap_or_default()+ .iter()+ .map(String::as_str)+ .collect::<Vec<_>>()+ && audio+ == event+ .audio+ .as_deref()+ .unwrap_or_default()+ .iter()+ .map(String::as_str)+ .collect::<Vec<_>>()+ && event.local_images.is_empty()+ && event.local_audio.is_empty()+}diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rsnew file mode 100644index 0000000000..8d0cb30fd9--- /dev/null+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs@@ -0,0 +1,191 @@+//! Mirrors the part of legacy cold resume that chooses the model-history checkpoint.+//!+//! Legacy rollback has two jobs. The forward history reducer decides which turns remain visible,+//! while cold resume scans backward to find the newest surviving compaction base. Migration needs+//! both answers because removing rollback markers must not change what the next resume sends to+//! the model.++use codex_protocol::protocol::EventMsg;+use codex_protocol::protocol::RolloutItem;++use super::rollback;++enum ReplayRecord {+ Compacted {+ record_index: usize,+ has_replacement_history: bool,+ },+ Rollback(u32),+ TurnStarted(String),+ TurnComplete(String),+ TurnAborted(Option<String>),+ TurnContext(Option<String>),+ UserBoundary,+}++#[derive(Default)]+struct ReverseSegment {+ turn_id: Option<String>,+ counts_as_user_turn: bool,+ compaction_index: Option<usize>,+}++/// Rewrite needed when legacy reverse replay skips a compaction segment but still resumes from+/// the suffix after that checkpoint.+pub(super) struct ModelReplayPlan {+ pub(super) empty_replacement_history_compaction: Option<usize>,+}++pub(super) struct ModelReplayPlanner {+ records: Vec<ReplayRecord>,+}++impl ModelReplayPlanner {+ pub(super) fn new() -> Self {+ Self {+ records: Vec::new(),+ }+ }++ pub(super) fn observe(&mut self, record_index: usize, item: &RolloutItem) {+ let record = match item {+ RolloutItem::Compacted(compacted) => ReplayRecord::Compacted {+ record_index,+ has_replacement_history: compacted.replacement_history.is_some(),+ },+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {+ ReplayRecord::Rollback(rollback.num_turns)+ }+ RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {+ ReplayRecord::TurnStarted(event.turn_id.clone())+ }+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {+ ReplayRecord::TurnComplete(event.turn_id.clone())+ }+ RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {+ ReplayRecord::TurnAborted(event.turn_id.clone())+ }+ RolloutItem::TurnContext(context) => ReplayRecord::TurnContext(context.turn_id.clone()),+ RolloutItem::EventMsg(EventMsg::UserMessage(_))+ | RolloutItem::InterAgentCommunication(_) => ReplayRecord::UserBoundary,+ RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(response) => {+ ReplayRecord::UserBoundary+ }+ RolloutItem::SessionMeta(_)+ | RolloutItem::ResponseItem(_)+ | RolloutItem::EventMsg(_)+ | RolloutItem::InterAgentCommunicationMetadata { .. }+ | RolloutItem::WorldState(_) => return,+ };+ self.records.push(record);+ }++ pub(super) fn finish(self) -> ModelReplayPlan {+ let mut selected_compaction = None;+ let mut suffix_start = None;+ let mut pending_rollback_turns = 0usize;+ let mut active_segment: Option<ReverseSegment> = None;++ for record in self.records.iter().rev() {+ match record {+ ReplayRecord::Compacted {+ record_index,+ has_replacement_history,+ } => {+ let segment = active_segment.get_or_insert_with(ReverseSegment::default);+ if segment.compaction_index.is_none() && *has_replacement_history {+ segment.compaction_index = Some(*record_index);+ suffix_start = record_index.checked_add(1);+ }+ }+ ReplayRecord::Rollback(num_turns) => {+ pending_rollback_turns = pending_rollback_turns+ .saturating_add(usize::try_from(*num_turns).unwrap_or(usize::MAX));+ }+ ReplayRecord::TurnComplete(turn_id) => {+ let segment = active_segment.get_or_insert_with(ReverseSegment::default);+ if segment.turn_id.is_none() {+ segment.turn_id = Some(turn_id.clone());+ }+ }+ ReplayRecord::TurnAborted(turn_id) => {+ if let Some(segment) = active_segment.as_mut() {+ if segment.turn_id.is_none() {+ segment.turn_id = turn_id.clone();+ }+ } else if turn_id.is_some() {+ active_segment = Some(ReverseSegment {+ turn_id: turn_id.clone(),+ ..Default::default()+ });+ }+ }+ ReplayRecord::TurnContext(turn_id) => {+ let segment = active_segment.get_or_insert_with(ReverseSegment::default);+ if segment.turn_id.is_none() {+ segment.turn_id = turn_id.clone();+ }+ }+ ReplayRecord::UserBoundary => {+ active_segment+ .get_or_insert_with(ReverseSegment::default)+ .counts_as_user_turn = true;+ }+ ReplayRecord::TurnStarted(turn_id) => {+ if active_segment.as_ref().is_some_and(|segment| {+ turn_ids_are_compatible(segment.turn_id.as_deref(), Some(turn_id.as_str()))+ }) && let Some(segment) = active_segment.take()+ {+ finalize_segment(+ segment,+ &mut selected_compaction,+ &mut pending_rollback_turns,+ );+ }+ }+ }+ }++ if let Some(segment) = active_segment {+ finalize_segment(+ segment,+ &mut selected_compaction,+ &mut pending_rollback_turns,+ );+ }++ // If reverse replay saw a replacement-history checkpoint but rolled back its segment,+ // it resumes from that checkpoint's suffix without using the checkpoint history. Keep an+ // empty checkpoint in the paginated file so older SQLite-visible records do not leak back+ // into model context after the rollback markers are gone.+ let empty_replacement_history_compaction = selected_compaction+ .is_none()+ .then_some(suffix_start)+ .flatten()+ .and_then(|start| start.checked_sub(1));+ ModelReplayPlan {+ empty_replacement_history_compaction,+ }+ }+}++fn finalize_segment(+ segment: ReverseSegment,+ selected_compaction: &mut Option<usize>,+ pending_rollback_turns: &mut usize,+) {+ if *pending_rollback_turns > 0 {+ if segment.counts_as_user_turn {+ *pending_rollback_turns -= 1;+ }+ return;+ }+ if selected_compaction.is_none() {+ *selected_compaction = segment.compaction_index;+ }+}++fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool {+ active_turn_id+ .is_none_or(|turn_id| item_turn_id.is_none_or(|item_turn_id| item_turn_id == turn_id))+}diff --git a/codex-rs/thread-store/src/local/rollout_migration/subagent.rs b/codex-rs/thread-store/src/local/rollout_migration/subagent.rsnew file mode 100644index 0000000000..54cfc75909--- /dev/null+++ b/codex-rs/thread-store/src/local/rollout_migration/subagent.rs@@ -0,0 +1,59 @@+//! Selects a smaller resume context for legacy subagent rollouts.+//!+//! Legacy subagents copied the parent's full rollout into every child rollout. Migrating those+//! files verbatim can preserve gigabytes of duplicated history even though a resumed subagent only+//! needs its latest bounded model context.+//!+//! This module reverse-scans the legacy rollout for a safe model-context checkpoint and returns the+//! smallest replay needed to resume from there. Malformed reverse-scanned records are skipped. If+//! we cannot prove that a bounded replay is safe, we return `None` and let the caller fall back to+//! full tolerant replay.++use std::fs::File;+use std::path::PathBuf;++use codex_protocol::protocol::RolloutItem;+use codex_protocol::protocol::SessionMetaLine;+use codex_rollout::ModelContextScan;+use codex_rollout::ModelContextScanProgress;+use codex_rollout::ReverseJsonlScanner;+use codex_rollout::ScanOutcome;+use serde_json::Value;++use super::line_parser;+use super::migration_error;+use crate::ThreadStoreResult;++/// Returns a bounded replay when the legacy child has enough durable context to reconstruct from+/// a suffix. Without that proof, the caller streams the complete legacy replay instead.+pub(super) async fn select_bounded_context(+ rollout_path: PathBuf,+ session_meta: SessionMetaLine,+) -> ThreadStoreResult<Option<Vec<RolloutItem>>> {+ tokio::task::spawn_blocking(move || {+ let file = File::open(rollout_path).map_err(migration_error)?;+ let mut scanner = ReverseJsonlScanner::new(file)+ .map_err(migration_error)?+ .with_max_record_bytes(super::MAX_ROLLOUT_LINE_BYTES);+ let mut scan = ModelContextScan::default();++ while let Some(outcome) = scanner.scan_next::<Value>().map_err(migration_error)? {+ let value = match outcome {+ ScanOutcome::Parsed(value) => value,+ ScanOutcome::Rejected(_) => continue,+ };+ let Ok(Some(line)) = line_parser::parse_legacy_rollout_value(value) else {+ continue;+ };+ if scan.push(line.item) == ModelContextScanProgress::Complete {+ let mut items = scan.finish(session_meta);+ items.retain(|item| !matches!(item, RolloutItem::SessionMeta(_)));+ return Ok(Some(items));+ }+ }++ Ok(None)+ })+ .await+ .map_err(migration_error)?+}diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rsindex dcdae5750e..7761ca382b 100644--- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs+++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs@@ -6,21 +6,36 @@ use std::path::PathBuf;#[cfg(unix)]use std::os::unix::fs::PermissionsExt;+use codex_app_server_protocol::build_turns_from_rollout_items;+use codex_protocol::AgentPath;use codex_protocol::ThreadId;+use codex_protocol::config_types::ReasoningSummary;+use codex_protocol::items::ReasoningItem;+use codex_protocol::items::TurnItem;use codex_protocol::models::ContentItem;use codex_protocol::models::ResponseItem;use codex_protocol::protocol::AgentMessageEvent;+use codex_protocol::protocol::AskForApproval;+use codex_protocol::protocol::CompactedItem;use codex_protocol::protocol::EventMsg;+use codex_protocol::protocol::InterAgentCommunication;+use codex_protocol::protocol::ItemCompletedEvent;use codex_protocol::protocol::RolloutItem;use codex_protocol::protocol::RolloutLine;+use codex_protocol::protocol::SandboxPolicy;use codex_protocol::protocol::SessionMeta;use codex_protocol::protocol::SessionMetaLine;use codex_protocol::protocol::SessionSource;use codex_protocol::protocol::SubAgentSource;use codex_protocol::protocol::ThreadHistoryMode;+use codex_protocol::protocol::ThreadRolledBackEvent;+use codex_protocol::protocol::TurnCompleteEvent;+use codex_protocol::protocol::TurnContextItem;+use codex_protocol::protocol::TurnStartedEvent;use codex_protocol::protocol::UserMessageEvent;use codex_rollout::RolloutConfig;use pretty_assertions::assert_eq;+use serde_json::json;use tempfile::TempDir;use super::LocalThreadStore;@@ -32,9 +47,15 @@ use super::decompress_rollout_to_path;use super::migration_journal_path;use super::thread_history;use super::write_migration_journal;+use crate::ItemSortKey;+use crate::ListItemsParams;use crate::ListTurnsParams;+use crate::LoadThreadHistoryParams;use crate::SortDirection;use crate::StoredTurnItemsView;+use crate::ThreadStore;+use crate::ThreadStoreError;+use crate::TurnPage;use crate::local::test_support::test_config;const TIMESTAMP: &str = "2025-01-03T12:00:00Z";@@ -92,6 +113,26 @@ fn write_rollout_with_fork(path}+fn move_to_archived(home: &Path, path: PathBuf) -> PathBuf {+ let directory = home.join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR);+ fs::create_dir_all(&directory).expect("create archived rollout directory");+ let archived_path = directory.join(path.file_name().expect("rollout filename"));+ fs::rename(path, &archived_path).expect("archive rollout");+ archived_path+}++fn compress_rollout(path: &Path) -> PathBuf {+ let compressed_path = path.with_extension("jsonl.zst");+ let compressed = zstd::stream::encode_all(+ fs::File::open(path).expect("open rollout"),+ /*level*/ 3,+ )+ .expect("compress rollout");+ fs::write(&compressed_path, compressed).expect("write compressed rollout");+ fs::remove_file(path).expect("remove plain rollout");+ compressed_path+}+fn user_message(text: &str) -> RolloutItem {RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {message: text.to_string(),@@ -107,6 +148,88 @@ fn agent_message(text: &str) -> RolloutItem {}))}+fn input_response_message(role: &str, text: &str) -> ResponseItem {+ ResponseItem::Message {+ id: None,+ role: role.to_string(),+ content: vec![ContentItem::InputText {+ text: text.to_string(),+ }],+ phase: None,+ internal_chat_message_metadata_passthrough: None,+ }+}++fn exec_completion(turn_id: &str, call_id: &str) -> RolloutItem {+ serde_json::from_value(json!({+ "type": "event_msg",+ "payload": {+ "type": "exec_command_end",+ "call_id": call_id,+ "turn_id": turn_id,+ "command": ["echo", "ok"],+ "cwd": "file:///tmp",+ "parsed_cmd": [],+ "source": "agent",+ "stdout": "ok",+ "stderr": "",+ "aggregated_output": "ok",+ "exit_code": 0,+ "duration": {"secs": 0, "nanos": 0},+ "formatted_output": "ok",+ "status": "completed"+ }+ }))+ .expect("build legacy exec completion")+}++fn item_completed(turn_id: &str, item_id: &str) -> RolloutItem {+ RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {+ thread_id: ThreadId::new(),+ turn_id: turn_id.to_string(),+ item: TurnItem::Reasoning(ReasoningItem {+ id: item_id.to_string(),+ summary_text: vec!["summary".to_string()],+ raw_content: Vec::new(),+ }),+ started_at_ms: None,+ completed_at_ms: 1_735_905_601_000,+ }))+}++fn started(turn_id: &str) -> RolloutItem {+ RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {+ turn_id: turn_id.to_string(),+ trace_id: None,+ started_at: Some(1_735_905_600),+ model_context_window: None,+ collaboration_mode_kind: Default::default(),+ }))+}++fn compacted(replacement_history: Vec<ResponseItem>) -> RolloutItem {+ RolloutItem::Compacted(CompactedItem {+ message: "checkpoint".to_string(),+ replacement_history: Some(replacement_history),+ window_number: Some(1),+ first_window_id: None,+ previous_window_id: None,+ window_id: None,+ })+}++fn completed(turn_id: &str) -> RolloutItem {+ RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {+ turn_id: turn_id.to_string(),+ last_agent_message: None,+ error: None,+ started_at: Some(1_735_905_600),+ completed_at: Some(1_735_905_601),+ duration_ms: Some(1_000),+ time_to_first_token_ms: None,+ }))+}+fn read_rollout(path: &Path) -> Vec<RolloutLine> {fs::read_to_string(path).expect("read migrated rollout")@@ -115,15 +238,6 @@ fn read_rollout(path: &Path) -> Vec<RolloutLine> {.collect()}-fn compress_rollout(path: PathBuf) -> PathBuf {- let compressed_path = path.with_extension("jsonl.zst");- let input = fs::File::open(&path).expect("open rollout for compression");- let compressed = zstd::stream::encode_all(input, /*level*/ 3).expect("compress rollout");- fs::write(&compressed_path, compressed).expect("write compressed rollout");- fs::remove_file(path).expect("remove plain rollout");- compressed_path-}-fn apply_options() -> RolloutMigrationOptions {RolloutMigrationOptions {mode: RolloutMigrationMode::Apply,@@ -147,6 +261,20 @@ async fn indexed_store(home: &Path) -> LocalThreadStore {LocalThreadStore::new(config, Some(state_db))}+async fn list_active_summary_turns(store: &LocalThreadStore, thread_id: ThreadId) -> TurnPage {+ store+ .list_turns(ListTurnsParams {+ thread_id,+ include_archived: false,+ cursor: None,+ page_size: 10,+ sort_direction: SortDirection::Asc,+ items_view: StoredTurnItemsView::Summary,+ })+ .await+ .expect("read projected turns")+}+#[tokio::test]async fn migration_publishes_canonical_projected_history_and_is_idempotent() {let home = TempDir::new().expect("create Codex home");@@ -189,6 +317,174 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {2);+ let turns = list_active_summary_turns(&store, thread_id).await;+ assert_eq!(turns.turns.len(), 1);+ assert_eq!(turns.turns[0].items.len(), 2);++ let bytes = fs::read(&path).expect("read first migration");+ let second = store+ .migrate_rollouts(apply_options())+ .await+ .expect("rerun migration");+ assert_eq!(+ second.outcomes[0].status,+ RolloutMigrationStatus::AlreadyPaginated+ );+ assert_eq!(fs::read(&path).expect("read idempotent rollout"), bytes);+}++#[tokio::test]+async fn migration_projects_explicit_and_implicit_legacy_completed_items() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let exec = exec_completion("explicit", "call-1");+ let reasoning = serde_json::from_value(json!({+ "type": "event_msg",+ "payload": {"type": "agent_reasoning", "text": "summary"}+ }))+ .expect("build legacy reasoning summary");+ let raw_reasoning = serde_json::from_value(json!({+ "type": "event_msg",+ "payload": {"type": "agent_reasoning_raw_content", "text": "raw"}+ }))+ .expect("build legacy reasoning content");+ write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("explicit"),+ exec,+ completed("explicit"),+ reasoning,+ raw_reasoning,+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate legacy completion events");++ let turns = list_active_summary_turns(&store, thread_id).await;+ assert_eq!(turns.turns.len(), 2);+ assert_eq!(turns.turns[0].turn_id, "explicit");+ let items = store+ .list_items(ListItemsParams {+ thread_id,+ turn_id: None,+ include_archived: false,+ cursor: None,+ page_size: 10,+ sort_direction: SortDirection::Asc,+ sort_key: ItemSortKey::CreatedAtOrdinal,+ after_updated_at_ordinal: None,+ })+ .await+ .expect("read projected items");+ assert_eq!(items.items.len(), 2);+ assert_eq!(items.items[0].turn_id, "explicit");+ assert_eq!(items.items[1].turn_id, turns.turns[1].turn_id);+ let command: serde_json::Value =+ serde_json::from_slice(&items.items[0].item_json).expect("parse projected command");+ let reasoning: serde_json::Value =+ serde_json::from_slice(&items.items[1].item_json).expect("parse projected reasoning");+ assert_eq!(command["type"], "commandExecution");+ assert_eq!(reasoning["type"], "reasoning");+ assert_eq!(reasoning["summary"], json!(["summary"]));+ assert_eq!(reasoning["content"], json!(["raw"]));+}++#[tokio::test]+async fn migration_keeps_late_completions_in_their_original_turn() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("old"),+ user_message("old question"),+ started("current"),+ user_message("current question"),+ exec_completion("old", "call-old"),+ agent_message("current answer"),+ completed("current"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate late completion");++ let started_turn_ids = read_rollout(&path)+ .into_iter()+ .filter_map(|line| match line.item {+ RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => Some(event.turn_id),+ _ => None,+ })+ .collect::<Vec<_>>();+ assert_eq!(started_turn_ids, vec!["old", "current"]);++ let items = store+ .list_items(ListItemsParams {+ thread_id,+ turn_id: None,+ include_archived: false,+ cursor: None,+ page_size: 10,+ sort_direction: SortDirection::Asc,+ sort_key: ItemSortKey::CreatedAtOrdinal,+ after_updated_at_ordinal: None,+ })+ .await+ .expect("read projected late completion");+ let item_turn_ids = items+ .items+ .iter()+ .map(|item| item.turn_id.as_str())+ .collect::<Vec<_>>();+ assert_eq!(item_turn_ids, vec!["old", "current", "old", "current"]);+}++#[tokio::test]+async fn migration_hoists_delayed_session_meta_before_paginated_history() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(home.path(), thread_id, SessionSource::Cli, Vec::new());+ let existing = fs::read_to_string(&path).expect("read legacy rollout");+ let pre_header = RolloutLine {+ timestamp: TIMESTAMP.to_string(),+ ordinal: None,+ item: user_message("before metadata"),+ };+ fs::write(+ &path,+ format!(+ "{}\n{existing}",+ serde_json::to_string(&pre_header).expect("serialize pre-header record")+ ),+ )+ .expect("write pre-header rollout");+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate delayed session metadata");++ let lines = read_rollout(&path);+ assert_eq!(lines[0].ordinal, Some(0));+ assert!(matches!(+ &lines[0].item,+ RolloutItem::SessionMeta(metadata)+ if metadata.meta.id == thread_id+ && metadata.meta.history_mode == ThreadHistoryMode::Paginated+ ));let turns = store.list_turns(ListTurnsParams {thread_id,@@ -196,14 +492,12 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {cursor: None,page_size: 10,sort_direction: SortDirection::Asc,- items_view: StoredTurnItemsView::Summary,+ items_view: StoredTurnItemsView::NotLoaded,}).await- .expect("read projected turns");+ .expect("read migrated turns");assert_eq!(turns.turns.len(), 1);- assert_eq!(turns.turns[0].items.len(), 2);- let bytes = fs::read(&path).expect("read first migration");let second = store.migrate_rollouts(apply_options()).await@@ -212,43 +506,918 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {second.outcomes[0].status,RolloutMigrationStatus::AlreadyPaginated);- assert_eq!(fs::read(&path).expect("read idempotent rollout"), bytes);}#[tokio::test]-async fn migration_preserves_valid_final_record_without_newline() {+async fn migration_preserves_valid_final_record_without_newline() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![user_message("question")],+ );+ let final_line = RolloutLine {+ timestamp: TIMESTAMP.to_string(),+ ordinal: None,+ item: agent_message("answer"),+ };+ fs::OpenOptions::new()+ .append(true)+ .open(&path)+ .expect("open legacy rollout")+ .write_all(+ serde_json::to_string(&final_line)+ .expect("serialize final record")+ .as_bytes(),+ )+ .expect("append final record");+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate legacy rollout");++ assert_eq!(+ read_rollout(&path)+ .iter()+ .filter(|line| matches!(line.item, RolloutItem::EventMsg(EventMsg::ItemCompleted(_))))+ .count(),+ 2+ );+}++#[tokio::test]+async fn migration_applies_historical_rollbacks_before_sqlite_projection() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("keep"),+ user_message("keep question"),+ agent_message("keep answer"),+ completed("keep"),+ started("remove"),+ user_message("remove question"),+ agent_message("remove answer"),+ completed("remove"),+ started("shell"),+ completed("shell"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ started("replacement"),+ user_message("replacement question"),+ agent_message("replacement answer"),+ completed("replacement"),+ ],+ );+ let store = indexed_store(home.path()).await;++ let report = store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate rolled-back thread");+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);++ let lines = read_rollout(&path);+ assert!(!lines.iter().any(|line| matches!(+ line.item,+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))+ )));+ let turns = list_active_summary_turns(&store, thread_id).await;+ assert_eq!(+ turns+ .turns+ .iter()+ .map(|turn| turn.turn_id.as_str())+ .collect::<Vec<_>>(),+ vec!["keep", "replacement"]+ );+}++#[tokio::test]+async fn migration_rolls_back_response_and_inter_agent_user_boundaries() {+ let home = TempDir::new().expect("create Codex home");+ let response_thread_id = ThreadId::new();+ let response_path = write_rollout(+ home.path(),+ response_thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::ResponseItem(input_response_message("user", "remove response boundary")),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ user_message("replacement question"),+ ],+ );+ let communication_thread_id = ThreadId::new();+ let communication_path = write_rollout(+ home.path(),+ communication_thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::InterAgentCommunication(InterAgentCommunication::new(+ AgentPath::root(),+ AgentPath::root().join("worker").expect("worker path"),+ Vec::new(),+ "remove communication boundary".to_string(),+ /*trigger_turn*/ true,+ )),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ user_message("replacement question"),+ ],+ );+ let contextual_thread_id = ThreadId::new();+ let contextual_path = write_rollout(+ home.path(),+ contextual_thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::ResponseItem(input_response_message("user", "keep first boundary")),+ RolloutItem::ResponseItem(input_response_message(+ "developer",+ "<permissions instructions>context only</permissions instructions>",+ )),+ RolloutItem::ResponseItem(input_response_message(+ "user",+ "<environment_context>context only</environment_context>",+ )),+ RolloutItem::ResponseItem(input_response_message("user", "remove real user boundary")),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate legacy rollback boundaries");++ assert!(+ !read_rollout(&response_path)+ .iter()+ .any(|line| matches!(line.item, RolloutItem::ResponseItem(_)))+ );+ assert!(+ !read_rollout(&communication_path)+ .iter()+ .any(|line| matches!(line.item, RolloutItem::InterAgentCommunication(_)))+ );+ assert_eq!(+ read_rollout(&contextual_path)+ .into_iter()+ .filter_map(|line| match line.item {+ RolloutItem::ResponseItem(response) => Some(response),+ _ => None,+ })+ .collect::<Vec<_>>(),+ vec![input_response_message("user", "keep first boundary")]+ );+}++#[tokio::test]+async fn migration_drops_trailing_context_when_rollback_arrives_before_next_turn() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::ResponseItem(input_response_message("user", "keep question")),+ RolloutItem::ResponseItem(input_response_message("user", "remove question")),+ RolloutItem::ResponseItem(input_response_message(+ "user",+ "<turn_aborted>remove this context too</turn_aborted>",+ )),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate trailing rollback context");++ assert_eq!(+ read_rollout(&path)+ .into_iter()+ .filter_map(|line| match line.item {+ RolloutItem::ResponseItem(response) => Some(response),+ _ => None,+ })+ .collect::<Vec<_>>(),+ vec![input_response_message("user", "keep question")]+ );+}++#[tokio::test]+async fn migration_coalesces_response_first_user_message_rollback_boundary() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::ResponseItem(input_response_message("user", "remove question")),+ user_message("remove question"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate response-first rollback");++ assert!(+ !read_rollout(&path)+ .iter()+ .any(|line| matches!(line.item, RolloutItem::ResponseItem(_)))+ );+}++#[tokio::test]+async fn migration_does_not_coalesce_distinct_adjacent_user_records() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::ResponseItem(input_response_message("user", "copied parent question")),+ user_message("child question"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate distinct adjacent user records");++ assert_eq!(+ read_rollout(&path)+ .into_iter()+ .filter_map(|line| match line.item {+ RolloutItem::ResponseItem(response) => Some(response),+ _ => None,+ })+ .collect::<Vec<_>>(),+ vec![input_response_message("user", "copied parent question")]+ );+}++#[tokio::test]+async fn migration_keeps_late_completions_for_surviving_turns_across_rollback() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("old"),+ user_message("old question"),+ started("remove"),+ user_message("remove question"),+ completed("old"),+ exec_completion("old", "call-old"),+ item_completed("old", "reason-old"),+ exec_completion("remove", "call-remove"),+ completed("remove"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ started("replacement"),+ user_message("replacement question"),+ completed("replacement"),+ ],+ );+ let legacy_turns = build_turns_from_rollout_items(+ &read_rollout(&path)+ .into_iter()+ .map(|line| line.item)+ .collect::<Vec<_>>(),+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate late completion rollback");++ let lines = read_rollout(&path);+ assert!(!lines.iter().any(|line| matches!(+ &line.item,+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event))+ if event.item.id() == "call-remove"+ )));+ assert!(lines.iter().any(|line| matches!(+ &line.item,+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event))+ if event.turn_id == "old" && event.item.id() == "call-old"+ )));+ assert!(lines.iter().any(|line| matches!(+ &line.item,+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event))+ if event.turn_id == "old" && event.item.id() == "reason-old"+ )));+ assert!(lines.iter().any(|line| matches!(+ &line.item,+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) if event.turn_id == "old"+ )));+ assert!(!lines.iter().any(|line| matches!(+ &line.item,+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) if event.turn_id == "remove"+ )));+ let turns = store+ .list_turns(ListTurnsParams {+ thread_id,+ include_archived: false,+ cursor: None,+ page_size: 10,+ sort_direction: SortDirection::Asc,+ items_view: StoredTurnItemsView::Summary,+ })+ .await+ .expect("read late-completion rollback turns");+ let turn_ids = turns+ .turns+ .iter()+ .map(|turn| turn.turn_id.as_str())+ .collect::<Vec<_>>();+ assert_eq!(+ turn_ids,+ legacy_turns+ .iter()+ .map(|turn| turn.id.as_str())+ .collect::<Vec<_>>()+ );+ let items = store+ .list_items(ListItemsParams {+ thread_id,+ turn_id: None,+ include_archived: false,+ cursor: None,+ page_size: 20,+ sort_direction: SortDirection::Asc,+ sort_key: ItemSortKey::CreatedAtOrdinal,+ after_updated_at_ordinal: None,+ })+ .await+ .expect("read late-completion rollback items");+ for legacy_turn in &legacy_turns {+ for legacy_item in &legacy_turn.items {+ if legacy_item.id().starts_with("item-") {+ continue;+ }+ assert_eq!(+ items+ .items+ .iter()+ .find(|item| item.item_id == legacy_item.id())+ .map(|item| item.turn_id.as_str()),+ Some(legacy_turn.id.as_str())+ );+ }+ }+}++#[tokio::test]+async fn migration_rolls_back_inter_agent_metadata_with_its_delivery() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let delivery = InterAgentCommunication::new(+ AgentPath::root(),+ AgentPath::root().join("worker").expect("worker path"),+ Vec::new(),+ "remove delivery".to_string(),+ /*trigger_turn*/ true,+ );+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::InterAgentCommunicationMetadata { trigger_turn: true },+ RolloutItem::ResponseItem(delivery.to_model_input_item()),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate rolled-back inter-agent delivery");++ assert!(!read_rollout(&path).iter().any(|line| {+ matches!(+ line.item,+ RolloutItem::InterAgentCommunicationMetadata { .. }+ | RolloutItem::ResponseItem(ResponseItem::AgentMessage { .. })+ )+ }));+}++#[tokio::test]+async fn migration_rolls_back_pre_compaction_turns_from_sqlite_history() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("keep-before-compaction"),+ user_message("old question"),+ completed("keep-before-compaction"),+ compacted(vec![+ input_response_message("user", "old question"),+ ResponseItem::Message {+ id: None,+ role: "assistant".to_string(),+ content: vec![ContentItem::OutputText {+ text: "old answer".to_string(),+ }],+ phase: None,+ internal_chat_message_metadata_passthrough: None,+ },+ ]),+ started("remove-after-compaction"),+ user_message("new question"),+ completed("remove-after-compaction"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 2,+ })),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate rollback through compaction");++ let turns = store+ .list_turns(ListTurnsParams {+ thread_id,+ include_archived: false,+ cursor: None,+ page_size: 10,+ sort_direction: SortDirection::Asc,+ items_view: StoredTurnItemsView::Summary,+ })+ .await+ .expect("read rollback-through-compaction turns");+ assert_eq!(turns.turns.len(), 1);+ let replacement_history = read_rollout(&path)+ .into_iter()+ .find_map(|line| match line.item {+ RolloutItem::Compacted(item) => item.replacement_history,+ _ => None,+ })+ .expect("retained compaction");+ assert!(replacement_history.is_empty());+}++#[tokio::test]+async fn migration_preserves_reverse_replay_anchor_after_pre_compaction_rollback() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ RolloutItem::ResponseItem(input_response_message("user", "remove question")),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ compacted(vec![input_response_message("user", "old question")]),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate pre-compaction rollback");++ let replacement_history = read_rollout(&path)+ .into_iter()+ .find_map(|line| match line.item {+ RolloutItem::Compacted(item) => item.replacement_history,+ _ => None,+ })+ .expect("retained compaction");+ assert!(replacement_history.is_empty());+}++#[tokio::test]+async fn migration_keeps_empty_replay_anchor_from_rolled_back_turn() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("keep"),+ user_message("keep question"),+ completed("keep"),+ started("remove"),+ user_message("remove question"),+ compacted(vec![input_response_message("user", "old question")]),+ completed("remove"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ started("replacement"),+ user_message("replacement question"),+ completed("replacement"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate rolled-back replay anchor");++ let replacement_history = read_rollout(&path)+ .into_iter()+ .find_map(|line| match line.item {+ RolloutItem::Compacted(item) => item.replacement_history,+ _ => None,+ })+ .expect("retained compaction");+ assert!(replacement_history.is_empty());+}++#[tokio::test]+async fn migration_uses_turn_context_to_select_reverse_replay_anchor() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("keep"),+ user_message("keep question"),+ compacted(vec![input_response_message("user", "keep question")]),+ completed("keep"),+ started("remove"),+ user_message("remove question"),+ serde_json::from_value(json!({+ "type": "event_msg",+ "payload": {+ "type": "turn_aborted",+ "turn_id": "other",+ "reason": "interrupted"+ }+ }))+ .expect("build mismatched turn abort"),+ serde_json::from_value(json!({+ "type": "turn_context",+ "payload": {+ "turn_id": "remove",+ "cwd": home.path(),+ "approval_policy": "never",+ "sandbox_policy": {"type": "read-only"},+ "model": "test-model",+ "summary": "auto"+ }+ }))+ .expect("build turn context"),+ compacted(vec![input_response_message("user", "remove question")]),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ started("replacement"),+ user_message("replacement question"),+ completed("replacement"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate turn-context replay anchor");++ let replacement_histories = read_rollout(&path)+ .into_iter()+ .filter_map(|line| match line.item {+ RolloutItem::Compacted(item) => item.replacement_history,+ _ => None,+ })+ .collect::<Vec<_>>();+ assert_eq!(+ replacement_histories,+ vec![vec![input_response_message("user", "keep question")]]+ );+}++#[tokio::test]+async fn migration_applies_cumulative_and_overflowing_rollbacks() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ started("first"),+ user_message("first question"),+ completed("first"),+ started("second"),+ user_message("second question"),+ completed("second"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 1,+ })),+ started("third"),+ user_message("third question"),+ completed("third"),+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {+ num_turns: 99,+ })),+ user_message("replacement question"),+ ],+ );+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate cumulative rollbacks");++ let turns = store+ .list_turns(ListTurnsParams {+ thread_id,+ include_archived: false,+ cursor: None,+ page_size: 10,+ sort_direction: SortDirection::Asc,+ items_view: StoredTurnItemsView::Summary,+ })+ .await+ .expect("read cumulative rollback turns");+ assert_eq!(turns.turns.len(), 1);+ assert!(!read_rollout(&path).iter().any(|line| matches!(+ line.item,+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))+ )));+}++#[tokio::test]+async fn migration_drops_copied_user_fork_metadata_without_creating_a_history_base() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let parent_id = ThreadId::new();+ let copied_metadata = SessionMeta {+ session_id: parent_id.into(),+ id: parent_id,+ timestamp: TIMESTAMP.to_string(),+ cwd: home.path().to_path_buf(),+ source: SessionSource::Cli,+ ..SessionMeta::default()+ };+ let copied_response =+ RolloutItem::ResponseItem(input_response_message("user", "copied parent history"));+ let path = write_rollout_with_fork(+ home.path(),+ thread_id,+ SessionSource::Cli,+ Some(parent_id),+ vec![+ RolloutItem::SessionMeta(SessionMetaLine {+ meta: copied_metadata,+ git: None,+ }),+ copied_response,+ user_message("child question"),+ agent_message("child answer"),+ ],+ );+ let expected_responses = read_rollout(&path)+ .into_iter()+ .filter_map(|line| match line.item {+ RolloutItem::ResponseItem(item) => {+ Some(serde_json::to_value(item).expect("serialize copied response"))+ }+ _ => None,+ })+ .collect::<Vec<_>>();+ let store = indexed_store(home.path()).await;++ let report = store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate copied user fork");++ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);+ let lines = read_rollout(&path);+ assert!(matches!(+ &lines[0].item,+ RolloutItem::SessionMeta(metadata)+ if metadata.meta.id == thread_id+ && metadata.meta.forked_from_id == Some(parent_id)+ && metadata.meta.history_mode == ThreadHistoryMode::Paginated+ && metadata.meta.history_base.is_none()+ ));+ assert_eq!(+ lines+ .iter()+ .filter(|line| matches!(line.item, RolloutItem::SessionMeta(_)))+ .count(),+ 1+ );+ assert_eq!(+ lines+ .into_iter()+ .filter_map(|line| match line.item {+ RolloutItem::ResponseItem(item) => {+ Some(serde_json::to_value(item).expect("serialize migrated response"))+ }+ _ => None,+ })+ .collect::<Vec<_>>(),+ expected_responses+ );+}++#[tokio::test]+async fn migration_compacts_subagent_prefix_and_does_not_project_it() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::SubAgent(SubAgentSource::Other("test".to_string())),+ vec![+ RolloutItem::Compacted(CompactedItem {+ message: "superseded checkpoint".repeat(1024),+ replacement_history: Some(Vec::new()),+ window_number: Some(1),+ first_window_id: None,+ previous_window_id: None,+ window_id: None,+ }),+ RolloutItem::Compacted(CompactedItem {+ message: "latest checkpoint".to_string(),+ replacement_history: Some(vec![ResponseItem::Message {+ id: None,+ role: "user".to_string(),+ content: vec![ContentItem::InputText {+ text: "latest compacted context".to_string(),+ }],+ phase: None,+ internal_chat_message_metadata_passthrough: None,+ }]),+ window_number: Some(2),+ first_window_id: None,+ previous_window_id: None,+ window_id: None,+ }),+ started("child-turn"),+ RolloutItem::TurnContext(TurnContextItem {+ turn_id: Some("child-turn".to_string()),+ cwd: serde_json::from_value(json!(home.path())).expect("absolute cwd"),+ workspace_roots: None,+ current_date: None,+ timezone: None,+ approval_policy: AskForApproval::Never,+ approvals_reviewer: None,+ sandbox_policy: SandboxPolicy::new_read_only_policy(),+ permission_profile: None,+ network: None,+ file_system_sandbox_policy: None,+ model: "test-model".to_string(),+ comp_hash: None,+ personality: None,+ collaboration_mode: None,+ multi_agent_version: None,+ multi_agent_mode: None,+ realtime_active: None,+ effort: None,+ summary: ReasoningSummary::Auto,+ }),+ user_message("child question"),+ agent_message("child answer"),+ completed("child-turn"),+ ],+ );+ writeln!(+ fs::OpenOptions::new()+ .append(true)+ .open(&path)+ .expect("open legacy subagent rollout"),+ "{{not valid rollout json"+ )+ .expect("append malformed record");+ let store = indexed_store(home.path()).await;++ store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate legacy subagent");++ let lines = read_rollout(&path);+ let RolloutItem::SessionMeta(metadata) = &lines[0].item else {+ panic!("migrated rollout should start with session metadata");+ };+ assert_eq!(+ metadata.meta.subagent_history_start_ordinal,+ Some(lines.len() as u64)+ );+ assert!(+ !fs::read_to_string(&path)+ .expect("read migrated rollout")+ .contains("superseded checkpoint")+ );+ let context = store+ .load_latest_model_context(LoadThreadHistoryParams {+ thread_id,+ include_archived: false,+ })+ .await+ .expect("load migrated model context");+ assert!(context.items.iter().any(|item| {+ matches!(item, RolloutItem::Compacted(compacted) if compacted.message == "latest checkpoint")+ }));+ assert!(+ list_active_summary_turns(&store, thread_id)+ .await+ .turns+ .is_empty()+ );+}++#[tokio::test]+async fn migration_keeps_small_uncompacted_subagent_replay_as_prefix() {let home = TempDir::new().expect("create Codex home");let thread_id = ThreadId::new();let path = write_rollout(home.path(),thread_id,- SessionSource::Cli,- vec![user_message("question")],+ SessionSource::SubAgent(SubAgentSource::Other("test".to_string())),+ vec![+ user_message("child question"),+ agent_message("child answer"),+ ],);- let final_line = RolloutLine {- timestamp: TIMESTAMP.to_string(),- ordinal: None,- item: agent_message("answer"),- };- fs::OpenOptions::new()- .append(true)- .open(&path)- .expect("open legacy rollout")- .write_all(- serde_json::to_string(&final_line)- .expect("serialize final record")- .as_bytes(),- )- .expect("append final record");let store = indexed_store(home.path()).await;store.migrate_rollouts(apply_options()).await- .expect("migrate legacy rollout");+ .expect("migrate uncompacted legacy subagent");+ let lines = read_rollout(&path);+ let RolloutItem::SessionMeta(metadata) = &lines[0].item else {+ panic!("migrated rollout should start with session metadata");+ };assert_eq!(- read_rollout(&path)+ metadata.meta.subagent_history_start_ordinal,+ Some(lines.len() as u64)+ );+ assert_eq!(+ lines.iter().filter(|line| matches!(line.item, RolloutItem::EventMsg(EventMsg::ItemCompleted(_)))).count(),@@ -257,90 +1426,39 @@ async fn migration_preserves_valid_final_record_without_newline() {}#[tokio::test]-async fn migration_preserves_copied_user_fork_history_without_creating_a_history_base() {+async fn migration_projects_memory_consolidation_as_ordinary_history() {let home = TempDir::new().expect("create Codex home");let thread_id = ThreadId::new();- let parent_id = ThreadId::new();- let copied_metadata = SessionMeta {- session_id: parent_id.into(),- id: parent_id,- timestamp: TIMESTAMP.to_string(),- cwd: home.path().to_path_buf(),- source: SessionSource::Cli,- ..SessionMeta::default()- };- let copied_response = RolloutItem::ResponseItem(ResponseItem::Message {- id: None,- role: "user".to_string(),- content: vec![ContentItem::InputText {- text: "copied parent history".to_string(),- }],- phase: None,- internal_chat_message_metadata_passthrough: None,- });- let path = write_rollout_with_fork(+ let path = write_rollout(home.path(),thread_id,- SessionSource::Cli,- Some(parent_id),+ SessionSource::SubAgent(SubAgentSource::MemoryConsolidation),vec![- RolloutItem::SessionMeta(SessionMetaLine {- meta: copied_metadata,- git: None,- }),- copied_response,- user_message("child question"),- agent_message("child answer"),+ user_message("memory question"),+ agent_message("memory answer"),],);- let expected_responses = read_rollout(&path)- .into_iter()- .filter_map(|line| match line.item {- RolloutItem::ResponseItem(item) => {- Some(serde_json::to_value(item).expect("serialize copied response"))- }- _ => None,- })- .collect::<Vec<_>>();let store = indexed_store(home.path()).await;- let report = store+ store.migrate_rollouts(apply_options()).await- .expect("migrate copied user fork");+ .expect("migrate memory consolidation rollout");- assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);let lines = read_rollout(&path);assert!(matches!(&lines[0].item,RolloutItem::SessionMeta(metadata)- if metadata.meta.id == thread_id- && metadata.meta.forked_from_id == Some(parent_id)- && metadata.meta.history_mode == ThreadHistoryMode::Paginated- && metadata.meta.history_base.is_none()- ));- assert!(matches!(- &lines[1].item,- RolloutItem::SessionMeta(metadata)- if metadata.meta.id == parent_id- && metadata.meta.history_mode == ThreadHistoryMode::Legacy+ if metadata.meta.history_mode == ThreadHistoryMode::Paginated+ && metadata.meta.subagent_history_start_ordinal.is_none()));- assert_eq!(- lines- .into_iter()- .filter_map(|line| match line.item {- RolloutItem::ResponseItem(item) => {- Some(serde_json::to_value(item).expect("serialize migrated response"))- }- _ => None,- })- .collect::<Vec<_>>(),- expected_responses- );+ let turns = list_active_summary_turns(&store, thread_id).await;+ assert_eq!(turns.turns.len(), 1);+ assert_eq!(turns.turns[0].items.len(), 2);}#[tokio::test]-async fn dry_run_reports_newest_first_and_skips_subagents() {+async fn dry_run_reports_migration_order() {let home = TempDir::new().expect("create Codex home");let root_id = ThreadId::new();let root = write_rollout(@@ -349,6 +1467,11 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {SessionSource::Cli,vec![user_message("root question")],);+ let newest_directory = home.path().join("sessions/2025/01/04");+ fs::create_dir_all(&newest_directory).expect("create newest rollout directory");+ let newest_root = newest_directory.join(format!("rollout-2025-01-04T12-00-00-{root_id}.jsonl"));+ fs::rename(&root, &newest_root).expect("move root rollout to newest date");+ let root = newest_root;let subagent_id = ThreadId::new();let subagent = write_rollout(home.path(),@@ -356,13 +1479,40 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {SessionSource::SubAgent(SubAgentSource::Other("test".to_string())),vec![user_message("subagent question")],);+ let memory_id = ThreadId::new();+ let memory = write_rollout(+ home.path(),+ memory_id,+ SessionSource::SubAgent(SubAgentSource::MemoryConsolidation),+ vec![user_message("memory question")],+ );+ let memory_directory = home.path().join("sessions/2025/01/01");+ fs::create_dir_all(&memory_directory).expect("create memory rollout directory");+ let moved_memory = memory_directory.join(memory.file_name().expect("memory filename"));+ fs::rename(memory, &moved_memory).expect("move memory rollout");let compressed_id = ThreadId::new();- let compressed = compress_rollout(write_rollout(+ let oldest_directory = home.path().join("sessions/2025/01/02");+ fs::create_dir_all(&oldest_directory).expect("create oldest rollout directory");+ let source_compressed_plain = write_rollout(home.path(),compressed_id,SessionSource::Cli,vec![user_message("compressed question")],- ));+ );+ let compressed_plain =+ oldest_directory.join(format!("rollout-2025-01-02T12-00-00-{compressed_id}.jsonl"));+ fs::rename(source_compressed_plain, &compressed_plain).expect("move compressed rollout");+ let compressed = compress_rollout(&compressed_plain);+ let archived_id = ThreadId::new();+ let archived = move_to_archived(+ home.path(),+ write_rollout(+ home.path(),+ archived_id,+ SessionSource::Cli,+ vec![user_message("archived question")],+ ),+ );let original = fs::read(&root).expect("read original root rollout");let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);@@ -371,16 +1521,13 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {.await.expect("inspect legacy rollouts");- let mut expected = vec![+ let expected = vec![(root.clone(), root_id, RolloutMigrationStatus::Eligible),- (- subagent,- subagent_id,- RolloutMigrationStatus::SkippedSubagent,- ),+ (subagent, subagent_id, RolloutMigrationStatus::Eligible),(compressed, compressed_id, RolloutMigrationStatus::Eligible),+ (moved_memory, memory_id, RolloutMigrationStatus::Eligible),+ (archived, archived_id, RolloutMigrationStatus::Eligible),];- expected.sort_by(|(left, ..), (right, ..)| right.cmp(left));assert_eq!(report.outcomes@@ -394,27 +1541,46 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {expected,);assert_eq!(fs::read(&root).expect("read inspected rollout"), original);++ let selected = store+ .migrate_rollouts(RolloutMigrationOptions {+ thread_ids: vec![root_id],+ ..RolloutMigrationOptions::default()+ })+ .await+ .expect("inspect selected rollout");+ assert_eq!(selected.outcomes.len(), 1);+ assert_eq!(selected.outcomes[0].thread_id, Some(root_id));+ assert_eq!(+ selected.outcomes[0].status,+ RolloutMigrationStatus::Eligible+ );}#[tokio::test]async fn migration_preserves_compressed_rollouts_during_publish_and_recovery() {let home = TempDir::new().expect("create Codex home");let thread_id = ThreadId::new();- let compressed_path = compress_rollout(write_rollout(+ let path = write_rollout(home.path(),thread_id,SessionSource::Cli,- vec![user_message("compressed question")],- ));- let plain_path = compressed_path.with_extension("");+ vec![+ user_message("compressed question"),+ agent_message("compressed answer"),+ ],+ );+ let compressed_path = compress_rollout(&path);let store = indexed_store(home.path()).await;let report = store.migrate_rollouts(apply_options()).await.expect("migrate compressed rollout");+assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);- assert!(!plain_path.exists());+ assert!(!path.exists());+ assert!(compressed_path.exists());assert_eq!(codex_rollout::read_session_meta_line(&compressed_path).await@@ -423,6 +1589,9 @@ async fn migration_preserves_compressed_rollouts_during_publish_and_recovery() {.history_mode,ThreadHistoryMode::Paginated);+ let turns = list_active_summary_turns(&store, thread_id).await;+ assert_eq!(turns.turns.len(), 1);+ assert_eq!(turns.turns[0].items.len(), 2);thread_history::delete_thread(&store, thread_id).await@@ -442,16 +1611,60 @@ async fn migration_preserves_compressed_rollouts_during_publish_and_recovery() {);assert!(recovered.outcomes[0].bytes_processed > 0);assert!(compressed_path.exists());- assert!(!plain_path.exists());+ assert!(!path.exists());assert!(!journal_path.exists());}+#[tokio::test]+async fn migration_migrates_archived_rollouts_without_unarchiving_them() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let active_path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ user_message("archived question"),+ agent_message("archived answer"),+ ],+ );+ let archived_path = move_to_archived(home.path(), active_path.clone());+ let store = indexed_store(home.path()).await;++ let report = store+ .migrate_rollouts(apply_options())+ .await+ .expect("migrate archived rollout");++ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);+ assert!(!active_path.exists());+ assert!(archived_path.exists());+ assert!(matches!(+ &read_rollout(&archived_path)[0].item,+ RolloutItem::SessionMeta(metadata)+ if metadata.meta.history_mode == ThreadHistoryMode::Paginated+ ));+ let turns = store+ .list_turns(ListTurnsParams {+ thread_id,+ include_archived: true,+ cursor: None,+ page_size: 10,+ sort_direction: SortDirection::Asc,+ items_view: StoredTurnItemsView::Summary,+ })+ .await+ .expect("read archived projected turns");+ assert_eq!(turns.turns.len(), 1);+ assert_eq!(turns.turns[0].items.len(), 2);+}+#[cfg(unix)]#[tokio::test]async fn decompression_temporaries_are_owner_only() {let home = TempDir::new().expect("create Codex home");let thread_id = ThreadId::new();- let compressed_path = compress_rollout(write_rollout(+ let compressed_path = compress_rollout(&write_rollout(home.path(),thread_id,SessionSource::Cli,@@ -502,6 +1715,31 @@ async fn migration_skips_threads_with_an_active_writer() {assert_eq!(fs::read(&path).expect("read unmodified rollout"), original);}+#[tokio::test]+async fn migration_apply_conflicts_with_rollout_maintenance() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![user_message("maintenance question")],+ );+ let original = fs::read(&path).expect("read legacy rollout");+ let _maintenance_guard = codex_rollout::try_acquire_rollout_maintenance_lock(home.path())+ .expect("acquire rollout maintenance lock")+ .expect("claim rollout maintenance lock");+ let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);++ let error = store+ .migrate_rollouts(apply_options())+ .await+ .expect_err("reject concurrent rollout maintenance");++ assert!(matches!(error, ThreadStoreError::Conflict { .. }));+ assert_eq!(fs::read(&path).expect("read untouched rollout"), original);+}+#[tokio::test]async fn migration_recovers_a_published_rollout_with_missing_projection() {let home = TempDir::new().expect("create Codex home");@@ -619,49 +1857,193 @@ async fn migration_recovers_pending_rollouts_before_new_work() {}#[tokio::test]-async fn failed_migration_preserves_the_legacy_rollout_and_can_be_retried() {+async fn migration_recovers_a_compressed_published_rollout() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![+ user_message("recover compressed question"),+ agent_message("recover compressed answer"),+ ],+ );+ let store = indexed_store(home.path()).await;+ store+ .migrate_rollouts(apply_options())+ .await+ .expect("publish canonical rollout");+ thread_history::delete_thread(&store, thread_id)+ .await+ .expect("simulate interrupted projection");+ let state_db = store.state_db.as_ref().expect("state db");+ let mut legacy_metadata = state_db+ .get_thread(thread_id)+ .await+ .expect("read thread metadata")+ .expect("thread metadata");+ legacy_metadata.history_mode = ThreadHistoryMode::Legacy;+ state_db+ .delete_thread(thread_id)+ .await+ .expect("remove paginated thread metadata");+ assert!(+ state_db+ .insert_thread_if_absent(&legacy_metadata)+ .await+ .expect("restore legacy thread metadata")+ );+ let journal_path = migration_journal_path(home.path(), thread_id);+ write_migration_journal(&journal_path)+ .await+ .expect("simulate pending migration journal");+ let compressed_path = compress_rollout(&path);++ let report = store+ .migrate_rollouts(apply_options())+ .await+ .expect("recover compressed published rollout");++ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);+ assert!(!path.exists());+ assert!(compressed_path.exists());+ assert!(!journal_path.exists());+ assert_eq!(+ store+ .state_db+ .as_ref()+ .expect("state db")+ .get_thread(thread_id)+ .await+ .expect("read repaired thread metadata")+ .expect("thread metadata")+ .history_mode,+ ThreadHistoryMode::Paginated+ );+ let projection = thread_history::projection_state(&store, thread_id)+ .await+ .expect("read repaired projection")+ .expect("projection was rebuilt");+ assert_eq!(+ projection.next_byte_offset,+ zstd::stream::decode_all(fs::File::open(&compressed_path).expect("open rollout"))+ .expect("decompress rollout")+ .len() as u64+ );+}++#[tokio::test]+async fn migration_skips_oversized_jsonl_records() {let home = TempDir::new().expect("create Codex home");let thread_id = ThreadId::new();let path = write_rollout(home.path(),thread_id,SessionSource::Cli,- vec![user_message("recoverable question")],+ vec![user_message("kept question")],);- let valid_length = fs::metadata(&path)- .expect("read valid rollout length")- .len();+ let oversized_output = "x".repeat(super::MAX_ROLLOUT_LINE_BYTES);let mut file = fs::OpenOptions::new().append(true).open(&path).expect("open legacy rollout");- writeln!(file, "{{not valid rollout json").expect("append malformed record");+ writeln!(+ file,+ "{{\"timestamp\":\"{TIMESTAMP}\",\"type\":\"response_item\",\"payload\":{{\"type\":\"function_call_output\",\"call_id\":\"call-1\",\"output\":\"{oversized_output}\"}}}}"+ )+ .expect("write oversized rollout record");drop(file);- let original = fs::read(&path).expect("read malformed legacy rollout");let store = indexed_store(home.path()).await;let report = store.migrate_rollouts(apply_options()).await- .expect("report malformed legacy rollout");+ .expect("migrate oversized rollout record");++ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);+ assert!(+ fs::metadata(&path)+ .expect("read migrated rollout metadata")+ .len()+ < super::MAX_ROLLOUT_LINE_BYTES as u64+ );+ let turns = list_active_summary_turns(&store, thread_id).await;+ assert_eq!(turns.turns.len(), 1);+ assert_eq!(turns.turns[0].items.len(), 1);+}++#[tokio::test]+async fn migration_skips_empty_rollout_files() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let directory = home.path().join("sessions/2025/01/03");+ fs::create_dir_all(&directory).expect("create rollout directory");+ let path = directory.join(format!("rollout-2025-01-03T12-00-00-{thread_id}.jsonl"));+ fs::File::create(&path).expect("create empty rollout");+ let store = indexed_store(home.path()).await;++ let report = store+ .migrate_rollouts(apply_options())+ .await+ .expect("inspect empty rollout");- assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Failed);assert_eq!(- fs::read(&path).expect("read preserved legacy rollout"),- original+ report.outcomes[0].status,+ RolloutMigrationStatus::SkippedEmpty+ );+ assert_eq!(fs::metadata(&path).expect("read empty rollout").len(), 0);+ assert!(+ store+ .state_db+ .as_ref()+ .expect("state db")+ .get_thread(thread_id)+ .await+ .expect("read thread metadata")+ .is_none());+}- fs::OpenOptions::new()- .write(true)+#[tokio::test]+async fn migration_skips_malformed_lines_and_trailing_partial_tail() {+ let home = TempDir::new().expect("create Codex home");+ let thread_id = ThreadId::new();+ let path = write_rollout(+ home.path(),+ thread_id,+ SessionSource::Cli,+ vec![user_message("kept question")],+ );+ let mut file = fs::OpenOptions::new()+ .append(true).open(&path)- .expect("open repaired legacy rollout")- .set_len(valid_length)- .expect("remove malformed tail");- let retry = store+ .expect("open legacy rollout");+ writeln!(file, "{{not valid rollout json").expect("append malformed record");+ let later_valid_line = RolloutLine {+ timestamp: TIMESTAMP.to_string(),+ ordinal: None,+ item: agent_message("kept answer"),+ };+ writeln!(+ file,+ "{}",+ serde_json::to_string(&later_valid_line).expect("serialize later valid record")+ )+ .expect("append later valid record");+ file.write_all(br#"{"timestamp":"unterminated""#)+ .expect("append partial tail");+ drop(file);+ let store = indexed_store(home.path()).await;++ let report = store.migrate_rollouts(apply_options()).await- .expect("retry repaired legacy rollout");+ .expect("migrate malformed legacy rollout");- assert_eq!(retry.outcomes[0].status, RolloutMigrationStatus::Migrated);+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);assert!(!migration_journal_path(home.path(), thread_id).exists());+ let turns = list_active_summary_turns(&store, thread_id).await;+ assert_eq!(turns.turns.len(), 1);+ assert_eq!(turns.turns[0].items.len(), 2);}