Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Preserve legacy semantics during rollout migration (#37191)

Owen Linaac9f842473a
diff --git a/codex-rs/rollout/src/reverse_jsonl_scanner.rs b/codex-rs/rollout/src/reverse_jsonl_scanner.rs
index 71c5880f9a..70919a7e61 100644
--- a/codex-rs/rollout/src/reverse_jsonl_scanner.rs
+++ b/codex-rs/rollout/src/reverse_jsonl_scanner.rs
@@ -23,6 +23,8 @@ pub struct ReverseJsonlScanner<R> {
chunk_position: usize,
chunk: Vec<u8>,
record_reversed: Vec<u8>,
+ max_record_bytes: Option<usize>,
+ discarding_oversized_record: bool,
}
impl<R> ReverseJsonlScanner<R>
@@ -52,9 +54,17 @@ where
chunk_position: 0,
chunk: vec![0; READ_CHUNK_SIZE],
record_reversed: Vec::new(),
+ max_record_bytes: None,
+ discarding_oversized_record: false,
})
}
+ /// Skips records larger than the configured limit without buffering or parsing them.
+ pub fn with_max_record_bytes(mut self, max_record_bytes: usize) -> Self {
+ self.max_record_bytes = Some(max_record_bytes);
+ self
+ }
+
/// Scans the next nonblank record.
///
/// I/O failures are returned as [`Err`]. Invalid JSON records are returned as
@@ -66,6 +76,10 @@ where
loop {
if self.chunk_position == 0 {
if self.next_chunk_end == 0 {
+ if self.discarding_oversized_record {
+ self.discarding_oversized_record = false;
+ return Ok(None);
+ }
return Ok(self.finish_record());
}
@@ -79,14 +93,36 @@ where
let chunk = &self.chunk[..self.chunk_position];
if let Some(newline_position) = chunk.iter().rposition(|byte| *byte == b'\n') {
- self.record_reversed
- .extend(chunk[newline_position + 1..].iter().rev().copied());
+ let fragment = &chunk[newline_position + 1..];
+ if !self.discarding_oversized_record {
+ if self.max_record_bytes.is_some_and(|max_record_bytes| {
+ self.record_reversed.len().saturating_add(fragment.len()) > max_record_bytes
+ }) {
+ self.record_reversed.clear();
+ self.discarding_oversized_record = true;
+ } else {
+ self.record_reversed.extend(fragment.iter().rev().copied());
+ }
+ }
self.chunk_position = newline_position;
+ if self.discarding_oversized_record {
+ self.discarding_oversized_record = false;
+ continue;
+ }
if let Some(outcome) = self.finish_record() {
return Ok(Some(outcome));
}
} else {
- self.record_reversed.extend(chunk.iter().rev().copied());
+ if !self.discarding_oversized_record {
+ if self.max_record_bytes.is_some_and(|max_record_bytes| {
+ self.record_reversed.len().saturating_add(chunk.len()) > max_record_bytes
+ }) {
+ self.record_reversed.clear();
+ self.discarding_oversized_record = true;
+ } else {
+ self.record_reversed.extend(chunk.iter().rev().copied());
+ }
+ }
self.chunk_position = 0;
}
}
diff --git a/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rs b/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rs
index 72c471f5de..044fd68e7c 100644
--- a/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rs
+++ b/codex-rs/rollout/src/reverse_jsonl_scanner_tests.rs
@@ -68,6 +68,21 @@ not-json
Ok(())
}
+#[test]
+fn skips_records_over_the_configured_limit() -> std::io::Result<()> {
+ let oversized = record(&"x".repeat(128));
+ let input = format!(
+ "{}\n{}\n{}\n",
+ serde_json::to_string(&record("first"))?,
+ serde_json::to_string(&oversized)?,
+ serde_json::to_string(&record("third"))?
+ );
+ let mut scanner = ReverseJsonlScanner::new(Cursor::new(input.into_bytes()))?
+ .with_max_record_bytes(/*max_record_bytes*/ 32);
+
+ assert_records(&mut scanner, &["third", "first"])
+}
+
#[test]
fn accepts_valid_json_at_eof() -> std::io::Result<()> {
let input = b"{\"value\":\"first\"}\n{\"value\":\"second\"}";
diff --git a/codex-rs/thread-store/src/local/rollout_migration.rs b/codex-rs/thread-store/src/local/rollout_migration.rs
index e08c2461df..facfa952d5 100644
--- a/codex-rs/thread-store/src/local/rollout_migration.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration.rs
@@ -15,8 +15,11 @@ use std::time::Duration;
use chrono::DateTime;
use codex_app_server_protocol::project_rollout_line;
use codex_protocol::ThreadId;
+use codex_protocol::protocol::InternalSessionSource;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
+use codex_protocol::protocol::SessionSource;
+use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadSource;
use serde::Serialize;
@@ -39,6 +42,10 @@ mod canonicalizer;
mod legacy_event;
mod line_parser;
mod publish;
+mod rollback;
+mod rollback_plan;
+mod rollback_replay;
+mod subagent;
use canonicalizer::LegacyRolloutCanonicalizer;
use publish::compress_rollout_to_path;
@@ -48,13 +55,25 @@ use publish::decompressed_staged_rollout_path;
use publish::migration_journal_path;
use publish::pending_migration_thread_ids;
use publish::remove_file_if_present;
+use publish::rewrite_subagent_history_boundary;
+use publish::rewritten_staged_rollout_path;
use publish::staged_rollout_path;
use publish::sync_parent_directory;
use publish::write_migration_journal;
+use rollback_plan::RollbackPlan;
+use rollback_plan::RollbackPlanner;
const PROJECTION_BATCH_BYTES: u64 = 256 * 1024;
const MAX_ROLLOUT_LINE_BYTES: usize = 16 * 1024 * 1024;
+enum CanonicalizationAttempt {
+ Complete {
+ expected_length: u64,
+ expected_ordinal: u64,
+ },
+ NeedsRollbackPlan,
+}
+
struct CanonicalizationSource<'a> {
thread_id: ThreadId,
source_path: &'a Path,
@@ -99,7 +118,7 @@ pub enum RolloutMigrationStatus {
Eligible,
Migrated,
AlreadyPaginated,
- SkippedSubagent,
+ SkippedEmpty,
SkippedBusy,
Failed,
}
@@ -132,6 +151,12 @@ struct RolloutRecord {
byte_count: u64,
}
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum RolloutMigrationKind {
+ Ordinary,
+ Subagent,
+}
+
impl RolloutMigrationRateLimiter {
fn new(max_mib_per_second: u64) -> ThreadStoreResult<Self> {
let bytes_per_second = max_mib_per_second
@@ -166,7 +191,7 @@ impl RolloutMigrationRateLimiter {
}
impl LocalThreadStore {
- /// Inspect or migrate eligible legacy rollout files beneath active sessions.
+ /// Inspect or migrate eligible legacy rollout files beneath active and archived sessions.
pub async fn migrate_rollouts(
&self,
options: RolloutMigrationOptions,
@@ -186,6 +211,15 @@ impl LocalThreadStore {
let mut paths =
find_rollout_paths(&self.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR))
.await?;
+ paths.extend(
+ find_rollout_paths(
+ &self
+ .config
+ .codex_home
+ .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR),
+ )
+ .await?,
+ );
if options.mode == RolloutMigrationMode::Apply {
let pending_thread_ids = pending_migration_thread_ids(&self.config.codex_home).await?;
paths.sort_by_key(|path| {
@@ -196,138 +230,159 @@ impl LocalThreadStore {
let mut report = RolloutMigrationReport::default();
for path in paths {
- let metadata = match codex_rollout::read_session_meta_line(&path).await {
- Ok(metadata) => metadata,
- Err(error) => {
- let thread_id = thread_id_from_rollout_filename(&path);
- if matches_selection(&options.thread_ids, thread_id) {
- report.outcomes.push(RolloutMigrationOutcome {
- thread_id,
- rollout_path: path,
- status: RolloutMigrationStatus::Failed,
- bytes_processed: 0,
- message: Some(error.to_string()),
- });
- }
- continue;
- }
- };
- let thread_id = metadata.meta.id;
- if !matches_selection(&options.thread_ids, Some(thread_id)) {
- continue;
- }
- if metadata.meta.source.is_non_root_agent()
- || matches!(
- metadata.meta.thread_source,
- Some(ThreadSource::Subagent | ThreadSource::MemoryConsolidation)
- )
+ if let Some(outcome) = self
+ .migrate_rollout_path(path, &options, &mut limiter)
+ .await?
{
- report.outcomes.push(RolloutMigrationOutcome {
- thread_id: Some(thread_id),
- rollout_path: path,
- status: RolloutMigrationStatus::SkippedSubagent,
- bytes_processed: 0,
- message: None,
- });
- continue;
+ report.outcomes.push(outcome);
}
+ }
- let journal_path = migration_journal_path(&self.config.codex_home, thread_id);
- if metadata.meta.history_mode == ThreadHistoryMode::Paginated {
- let bytes_before = limiter.bytes_processed;
- let result = if options.mode == RolloutMigrationMode::Apply
- && tokio::fs::try_exists(&journal_path)
- .await
- .map_err(migration_error)?
- {
- match self
- .recover_published_migration(thread_id, &path, &journal_path, &mut limiter)
- .await
- {
- Ok(()) => Ok(RolloutMigrationStatus::Migrated),
- Err(ThreadStoreError::Conflict { message }) => {
- let bytes_processed =
- limiter.bytes_processed.saturating_sub(bytes_before);
- report.outcomes.push(skipped_busy_outcome(
- thread_id,
- path,
- message,
- bytes_processed,
- ));
- continue;
- }
- Err(error) => Err(error),
- }
- } else {
- Ok(RolloutMigrationStatus::AlreadyPaginated)
- };
- let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
- report
- .outcomes
- .push(migration_outcome(thread_id, path, result, bytes_processed));
- continue;
- }
+ Ok(report)
+ }
- if options.mode == RolloutMigrationMode::DryRun {
- report.outcomes.push(RolloutMigrationOutcome {
- thread_id: Some(thread_id),
+ async fn migrate_rollout_path(
+ &self,
+ mut path: PathBuf,
+ options: &RolloutMigrationOptions,
+ limiter: &mut RolloutMigrationRateLimiter,
+ ) -> ThreadStoreResult<Option<RolloutMigrationOutcome>> {
+ let metadata = match codex_rollout::read_session_meta_line(&path).await {
+ Ok(metadata) => metadata,
+ Err(error) => {
+ let thread_id = thread_id_from_rollout_filename(&path);
+ if !matches_selection(&options.thread_ids, thread_id) {
+ return Ok(None);
+ }
+ let empty = tokio::fs::metadata(&path)
+ .await
+ .is_ok_and(|metadata| metadata.len() == 0);
+ return Ok(Some(RolloutMigrationOutcome {
+ thread_id,
rollout_path: path,
- status: RolloutMigrationStatus::Eligible,
+ status: if empty {
+ RolloutMigrationStatus::SkippedEmpty
+ } else {
+ RolloutMigrationStatus::Failed
+ },
bytes_processed: 0,
- message: None,
- });
- continue;
+ message: (!empty).then(|| error.to_string()),
+ }));
}
+ };
+ let thread_id = metadata.meta.id;
+ if !matches_selection(&options.thread_ids, Some(thread_id)) {
+ return Ok(None);
+ }
+ let is_memory_consolidation = matches!(
+ metadata.meta.source,
+ SessionSource::Internal(InternalSessionSource::MemoryConsolidation)
+ | SessionSource::SubAgent(SubAgentSource::MemoryConsolidation)
+ ) || matches!(
+ metadata.meta.thread_source,
+ Some(ThreadSource::MemoryConsolidation)
+ );
+ let kind = if !is_memory_consolidation
+ && (matches!(metadata.meta.source, SessionSource::SubAgent(_))
+ || matches!(metadata.meta.thread_source, Some(ThreadSource::Subagent)))
+ {
+ RolloutMigrationKind::Subagent
+ } else {
+ RolloutMigrationKind::Ordinary
+ };
- let writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {
- Ok(guard) => guard,
- Err(ThreadStoreError::Conflict { message }) => {
- report.outcomes.push(skipped_busy_outcome(
- thread_id, path, message, /*bytes_processed*/ 0,
- ));
- continue;
- }
- Err(error) => {
- report.outcomes.push(migration_outcome(
- thread_id,
- path,
- Err(error),
- /*bytes_processed*/ 0,
- ));
- continue;
- }
- };
- let bytes_before = limiter.bytes_processed;
- let result = match self
- .migrate_one_rollout(thread_id, &path, &journal_path, &mut limiter)
+ let journal_path = migration_journal_path(&self.config.codex_home, thread_id);
+ let pending_published_migration = metadata.meta.history_mode
+ == ThreadHistoryMode::Paginated
+ && options.mode == RolloutMigrationMode::Apply
+ && tokio::fs::try_exists(&journal_path)
.await
- {
- Ok(()) => Ok(RolloutMigrationStatus::Migrated),
- Err(error) => {
- if let Err(cleanup_error) = self
- .cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)
- .await
- {
- Err(migration_error(format!(
- "{error}; failed to clean up unpublished migration: {cleanup_error}"
- )))
- } else {
- Err(error)
+ .map_err(migration_error)?;
+ if metadata.meta.history_mode == ThreadHistoryMode::Paginated {
+ let bytes_before = limiter.bytes_processed;
+ let result = if pending_published_migration {
+ match self
+ .recover_published_migration(thread_id, &path, &journal_path, limiter)
+ .await
+ {
+ Ok(recovered_path) => {
+ path = recovered_path;
+ Ok(RolloutMigrationStatus::Migrated)
+ }
+ Err(ThreadStoreError::Conflict { message }) => {
+ let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
+ return Ok(Some(skipped_busy_outcome(
+ thread_id,
+ path,
+ message,
+ bytes_processed,
+ )));
}
+ Err(error) => Err(error),
}
+ } else {
+ Ok(RolloutMigrationStatus::AlreadyPaginated)
};
- drop(writer_guard);
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
- let outcome = match result {
- Err(ThreadStoreError::Conflict { message }) => {
- skipped_busy_outcome(thread_id, path, message, bytes_processed)
- }
- result => migration_outcome(thread_id, path, result, bytes_processed),
- };
- report.outcomes.push(outcome);
+ return Ok(Some(migration_outcome(
+ thread_id,
+ path,
+ result,
+ bytes_processed,
+ )));
}
- Ok(report)
+ if options.mode == RolloutMigrationMode::DryRun {
+ return Ok(Some(migration_outcome(
+ thread_id,
+ path,
+ Ok(RolloutMigrationStatus::Eligible),
+ /*bytes_processed*/ 0,
+ )));
+ }
+
+ let _writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {
+ Ok(guard) => guard,
+ Err(ThreadStoreError::Conflict { message }) => {
+ return Ok(Some(skipped_busy_outcome(
+ thread_id, path, message, /*bytes_processed*/ 0,
+ )));
+ }
+ Err(error) => {
+ return Ok(Some(migration_outcome(
+ thread_id,
+ path,
+ Err(error),
+ /*bytes_processed*/ 0,
+ )));
+ }
+ };
+ let bytes_before = limiter.bytes_processed;
+ let result = match self
+ .migrate_one_rollout(thread_id, &path, &journal_path, kind, limiter)
+ .await
+ {
+ Ok(()) => Ok(RolloutMigrationStatus::Migrated),
+ Err(error) => {
+ if let Err(cleanup_error) = self
+ .cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)
+ .await
+ {
+ Err(migration_error(format!(
+ "{error}; failed to clean up unpublished migration: {cleanup_error}"
+ )))
+ } else {
+ Err(error)
+ }
+ }
+ };
+ let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
+ Ok(Some(match result {
+ Err(ThreadStoreError::Conflict { message }) => {
+ skipped_busy_outcome(thread_id, path, message, bytes_processed)
+ }
+ result => migration_outcome(thread_id, path, result, bytes_processed),
+ }))
}
async fn migrate_one_rollout(
@@ -335,6 +390,7 @@ impl LocalThreadStore {
thread_id: ThreadId,
rollout_path: &Path,
journal_path: &Path,
+ kind: RolloutMigrationKind,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<()> {
if let Some(state_db) = &self.state_db
@@ -403,8 +459,31 @@ impl LocalThreadStore {
source_permissions: &source_permissions,
canonical_session_meta: &canonical_session_meta,
};
- let (expected_length, expected_ordinal) =
- Self::write_canonical_rollout(&canonicalization_source, limiter).await?;
+ let bounded_subagent_context = if kind == RolloutMigrationKind::Subagent {
+ let RolloutItem::SessionMeta(session_meta) = &canonical_session_meta.item else {
+ return Err(migration_error("canonical session metadata is missing"));
+ };
+ let context =
+ subagent::select_bounded_context(source_path.to_path_buf(), session_meta.clone())
+ .await?;
+ limiter.account(source_metadata.len()).await;
+ context
+ } else {
+ None
+ };
+ let (_, expected_ordinal) = if let Some(items) = bounded_subagent_context {
+ Self::write_bounded_subagent_rollout(&canonicalization_source, items, limiter).await?
+ } else {
+ Self::write_rollout_with_rollback_plan(&canonicalization_source, limiter).await?
+ };
+
+ if kind == RolloutMigrationKind::Subagent {
+ rewrite_subagent_history_boundary(&staged_path, expected_ordinal).await?;
+ }
+ let expected_length = tokio::fs::metadata(&staged_path)
+ .await
+ .map_err(migration_error)?
+ .len();
// SQLite projection only starts after every staged-file mutation is durable.
let modified_at = source_modified;
@@ -473,10 +552,97 @@ impl LocalThreadStore {
.await
}
- async fn write_canonical_rollout(
+ async fn build_rollback_plan(
+ source_path: &Path,
+ limiter: &mut RolloutMigrationRateLimiter,
+ ) -> ThreadStoreResult<RollbackPlan> {
+ let source_file = File::open(source_path).await.map_err(migration_error)?;
+ let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);
+ let mut bytes = Vec::new();
+ let mut planner = RollbackPlanner::new();
+ while let Some(record) = read_rollout_record(&mut source, &mut bytes).await? {
+ limiter.account(record.byte_count).await;
+ if let Some(line) = record.line {
+ planner.observe(&line)?;
+ }
+ }
+ Ok(planner.finish())
+ }
+
+ async fn write_rollout_with_rollback_plan(
+ input: &CanonicalizationSource<'_>,
+ limiter: &mut RolloutMigrationRateLimiter,
+ ) -> ThreadStoreResult<(u64, u64)> {
+ let attempt = Self::write_canonical_rollout(input, /*plan*/ None, limiter).await?;
+ match attempt {
+ CanonicalizationAttempt::Complete {
+ expected_length,
+ expected_ordinal,
+ } => Ok((expected_length, expected_ordinal)),
+ CanonicalizationAttempt::NeedsRollbackPlan => {
+ remove_file_if_present(input.staged_path).await?;
+ let plan = Self::build_rollback_plan(input.source_path, limiter).await?;
+ let CanonicalizationAttempt::Complete {
+ expected_length,
+ expected_ordinal,
+ } = Self::write_canonical_rollout(input, Some(&plan), limiter).await?
+ else {
+ return Err(migration_error(
+ "planned rollout still contains a rollback marker",
+ ));
+ };
+ Ok((expected_length, expected_ordinal))
+ }
+ }
+ }
+
+ async fn write_bounded_subagent_rollout(
input: &CanonicalizationSource<'_>,
+ items: Vec<RolloutItem>,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<(u64, u64)> {
+ let staged_file = tokio::fs::OpenOptions::new()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(input.staged_path)
+ .await
+ .map_err(migration_error)?;
+ staged_file
+ .set_permissions(input.source_permissions.clone())
+ .await
+ .map_err(migration_error)?;
+ let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);
+ let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);
+ let written = canonicalizer
+ .write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)
+ .await?;
+ limiter.account(written).await;
+ for item in items {
+ let line = RolloutLine {
+ timestamp: input.canonical_session_meta.timestamp.clone(),
+ ordinal: None,
+ item,
+ };
+ let written = canonicalizer.process_line(line, &mut staged).await?;
+ limiter.account(written).await;
+ }
+ let written = canonicalizer
+ .finish(&mut staged, &input.canonical_session_meta.timestamp)
+ .await?;
+ limiter.account(written).await;
+ staged.flush().await.map_err(migration_error)?;
+ Ok((
+ canonicalizer.output_byte_offset(),
+ canonicalizer.next_ordinal(),
+ ))
+ }
+
+ async fn write_canonical_rollout(
+ input: &CanonicalizationSource<'_>,
+ plan: Option<&RollbackPlan>,
+ limiter: &mut RolloutMigrationRateLimiter,
+ ) -> ThreadStoreResult<CanonicalizationAttempt> {
let source_file = File::open(input.source_path)
.await
.map_err(migration_error)?;
@@ -494,6 +660,7 @@ impl LocalThreadStore {
let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);
let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);
let mut bytes = Vec::new();
+ let mut parsed_record_index = 0_usize;
let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);
let written = canonicalizer
.write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)
@@ -506,24 +673,45 @@ impl LocalThreadStore {
let Some(line) = record.line else {
continue;
};
- if matches!(
- &line.item,
- RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))
- ) {
- return Err(migration_error("legacy rollout contains a rollback marker"));
+ if plan.is_none()
+ && matches!(
+ &line.item,
+ RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))
+ )
+ {
+ return Ok(CanonicalizationAttempt::NeedsRollbackPlan);
}
+ let line = if let Some(plan) = plan {
+ let planned = plan.apply(parsed_record_index, line)?;
+ parsed_record_index = parsed_record_index
+ .checked_add(1)
+ .ok_or_else(|| migration_error("legacy rollout record index overflow"))?;
+ let Some(line) = planned else {
+ continue;
+ };
+ line
+ } else {
+ line
+ };
last_timestamp = line.timestamp.clone();
let written = canonicalizer.process_line(line, &mut staged).await?;
limiter.account(written).await;
}
+ if let Some(plan) = plan
+ && parsed_record_index != plan.record_count()
+ {
+ return Err(migration_error(
+ "rollback plan source length changed during replay",
+ ));
+ }
let written = canonicalizer.finish(&mut staged, &last_timestamp).await?;
limiter.account(written).await;
staged.flush().await.map_err(migration_error)?;
- Ok((
- canonicalizer.output_byte_offset(),
- canonicalizer.next_ordinal(),
- ))
+ Ok(CanonicalizationAttempt::Complete {
+ expected_length: canonicalizer.output_byte_offset(),
+ expected_ordinal: canonicalizer.next_ordinal(),
+ })
}
async fn recover_published_migration(
@@ -532,7 +720,7 @@ impl LocalThreadStore {
rollout_path: &Path,
journal_path: &Path,
limiter: &mut RolloutMigrationRateLimiter,
- ) -> ThreadStoreResult<()> {
+ ) -> ThreadStoreResult<PathBuf> {
let _writer_guard = self.writer_lock_coordinator.acquire(thread_id)?;
let decompressed_path = rollout_path_is_compressed(rollout_path)
.then(|| decompressed_staged_rollout_path(rollout_path))
@@ -572,7 +760,8 @@ impl LocalThreadStore {
remove_file_if_present(decompressed_path).await?;
}
self.finish_published_migration(thread_id, journal_path)
- .await
+ .await?;
+ Ok(rollout_path.to_path_buf())
}
async fn cleanup_failed_unpublished_migration(
@@ -589,7 +778,9 @@ impl LocalThreadStore {
}
thread_history::delete_thread(self, thread_id).await?;
- remove_file_if_present(&staged_rollout_path(rollout_path)?).await?;
+ let staged_path = staged_rollout_path(rollout_path)?;
+ remove_file_if_present(&staged_path).await?;
+ remove_file_if_present(&rewritten_staged_rollout_path(&staged_path)?).await?;
remove_file_if_present(&compressed_staged_rollout_path(rollout_path)?).await?;
remove_file_if_present(&decompressed_staged_rollout_path(rollout_path)?).await?;
remove_file_if_present(journal_path).await?;
@@ -623,6 +814,11 @@ impl LocalThreadStore {
rollout_path: &Path,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<()> {
+ let subagent_history_start_ordinal = codex_rollout::read_session_meta_line(rollout_path)
+ .await
+ .map_err(migration_error)?
+ .meta
+ .subagent_history_start_ordinal;
let file = File::open(rollout_path).await.map_err(migration_error)?;
let mut reader = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, file);
let mut line_bytes = Vec::new();
@@ -650,7 +846,11 @@ impl LocalThreadStore {
start_byte_offset: offset,
end_byte_offset: next_offset,
fallback_created_at_ms: Some(fallback_created_at_ms),
- changes: project_rollout_line(&line),
+ changes: if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) {
+ Default::default()
+ } else {
+ project_rollout_line(&line)
+ },
}));
offset = next_offset;
@@ -688,7 +888,7 @@ async fn read_rollout_record(
bytes: &mut Vec<u8>,
) -> ThreadStoreResult<Option<RolloutRecord>> {
bytes.clear();
- let byte_count = reader
+ let mut byte_count = reader
.take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)
.read_until(b'\n', bytes)
.await
@@ -697,11 +897,30 @@ async fn read_rollout_record(
return Ok(None);
}
if byte_count > MAX_ROLLOUT_LINE_BYTES {
- return Err(migration_error(
- "rollout contains an oversized JSONL record",
- ));
+ // Some historical tool outputs are too large to migrate safely in memory. Discard the
+ // whole record, including any unread suffix.
+ while bytes.last() != Some(&b'\n') {
+ bytes.clear();
+ let chunk_bytes = reader
+ .take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)
+ .read_until(b'\n', bytes)
+ .await
+ .map_err(migration_error)?;
+ if chunk_bytes == 0 {
+ break;
+ }
+ byte_count = byte_count
+ .checked_add(chunk_bytes)
+ .ok_or_else(|| migration_error("rollout record byte count overflow"))?;
+ }
+ return Ok(Some(RolloutRecord {
+ line: None,
+ byte_count: byte_count as u64,
+ }));
}
- let line = line_parser::parse_legacy_rollout_line(bytes).map_err(migration_error)?;
+ // Legacy records do not have ordinals, so malformed complete records cannot be repaired.
+ // Skip them and let the next newline-delimited record resynchronize the stream.
+ let line = line_parser::parse_legacy_rollout_line(bytes).unwrap_or(None);
Ok(Some(RolloutRecord {
line,
byte_count: byte_count as u64,
@@ -734,7 +953,15 @@ async fn find_rollout_paths(root: &Path) -> ThreadStoreResult<Vec<PathBuf>> {
if name.starts_with("rollout-")
&& (name.ends_with(".jsonl") || name.ends_with(".jsonl.zst"))
{
- paths.push(entry.path());
+ let path = entry.path();
+ if name.ends_with(".jsonl.zst")
+ && tokio::fs::try_exists(codex_rollout::plain_rollout_path(&path))
+ .await
+ .map_err(migration_error)?
+ {
+ continue;
+ }
+ paths.push(path);
}
}
}
@@ -747,6 +974,12 @@ fn matches_selection(selected: &[ThreadId], actual: Option<ThreadId>) -> bool {
selected.is_empty() || actual.is_some_and(|thread_id| selected.contains(&thread_id))
}
+fn rollout_path_is_compressed(path: &Path) -> bool {
+ path.file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| name.ends_with(".jsonl.zst"))
+}
+
fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {
let name = path.file_name()?.to_str()?;
let stem = name
@@ -756,12 +989,6 @@ fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {
ThreadId::from_string(stem.get(start..)?).ok()
}
-fn rollout_path_is_compressed(path: &Path) -> bool {
- path.file_name()
- .and_then(|name| name.to_str())
- .is_some_and(|name| name.ends_with(".jsonl.zst"))
-}
-
fn migration_outcome(
thread_id: ThreadId,
rollout_path: PathBuf,
diff --git a/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs b/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs
index d1519fe697..87ce548d5d 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs
@@ -1,8 +1,9 @@
//! Replays normalized legacy rollout records into canonical paginated JSONL.
//!
//! `line_parser` makes old JSON shapes parseable, `legacy_event` converts obsolete completion
-//! events into modern turn items. This module assigns stable ordinals, keeps `SessionMeta` at
-//! ordinal zero, and emits the surviving history selected by the caller.
+//! events into modern turn items. Rollback planning happens before this writer sees a record, so
+//! this module only assigns stable ordinals, keeps `SessionMeta` at ordinal zero, and emits the
+//! already-selected surviving history.
//!
//! The goal is to preserve the model-visible conversation, not to preserve every legacy record
//! byte-for-byte. Filesystem publishing and SQLite projection intentionally live outside this
@@ -52,7 +53,6 @@ pub(super) struct LegacyRolloutCanonicalizer {
active_turn: Option<ActiveTurn>,
known_turn_ids: HashSet<String>,
reasoning: Option<ReasoningItem>,
- saw_source_session_meta: bool,
}
impl LegacyRolloutCanonicalizer {
@@ -67,18 +67,17 @@ impl LegacyRolloutCanonicalizer {
active_turn: None,
known_turn_ids: HashSet::new(),
reasoning: None,
- saw_source_session_meta: false,
}
}
- pub(super) fn output_byte_offset(&self) -> u64 {
- self.output_byte_offset
- }
-
pub(super) fn next_ordinal(&self) -> u64 {
self.next_ordinal
}
+ pub(super) fn output_byte_offset(&self) -> u64 {
+ self.output_byte_offset
+ }
+
pub(super) async fn write_head_session_meta(
&mut self,
line: RolloutLine,
@@ -114,14 +113,7 @@ impl LegacyRolloutCanonicalizer {
let timestamp = line.timestamp;
let bytes_before = self.bytes_written;
match line.item {
- RolloutItem::SessionMeta(metadata) => {
- if !self.saw_source_session_meta {
- self.saw_source_session_meta = true;
- return Ok(0);
- }
- self.write_item(writer, &timestamp, RolloutItem::SessionMeta(metadata))
- .await?;
- }
+ RolloutItem::SessionMeta(_) => return Ok(0),
RolloutItem::ResponseItem(ResponseItem::Other) => {
return Err(migration_error(
"legacy rollout contains an unsupported response item",
diff --git a/codex-rs/thread-store/src/local/rollout_migration/line_parser.rs b/codex-rs/thread-store/src/local/rollout_migration/line_parser.rs
index 872e2a3e7e..dd71bcd8e5 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/line_parser.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/line_parser.rs
@@ -26,7 +26,11 @@ pub(super) fn parse_legacy_rollout_line(bytes: &[u8]) -> Result<Option<RolloutLi
// Deserializing through Value is intentional. Some historical numeric
// payloads fail serde_json's streaming enum path but deserialize correctly
// once their tagged object shape has been materialized.
- let mut value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
+ let value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
+ parse_legacy_rollout_value(value)
+}
+
+pub(super) fn parse_legacy_rollout_value(mut value: Value) -> Result<Option<RolloutLine>, String> {
if should_skip_retired_record(&value) {
return Ok(None);
}
diff --git a/codex-rs/thread-store/src/local/rollout_migration/publish.rs b/codex-rs/thread-store/src/local/rollout_migration/publish.rs
index 21f142e3c2..cd42249970 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/publish.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/publish.rs
@@ -22,6 +22,13 @@ use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::fs::PermissionsExt;
use codex_protocol::ThreadId;
+use codex_protocol::protocol::RolloutItem;
+use codex_protocol::protocol::RolloutLine;
+use tokio::fs::File;
+use tokio::io::AsyncBufReadExt;
+use tokio::io::AsyncWriteExt;
+use tokio::io::BufReader;
+use tokio::io::BufWriter;
use super::migration_error;
use crate::ThreadStoreResult;
@@ -139,6 +146,74 @@ pub(super) async fn compress_rollout_to_path(
.map_err(migration_error)
}
+pub(super) fn rewritten_staged_rollout_path(staged_path: &Path) -> ThreadStoreResult<PathBuf> {
+ let filename = staged_path
+ .file_name()
+ .and_then(|filename| filename.to_str())
+ .ok_or_else(|| migration_error("staged rollout path has no valid filename"))?;
+ Ok(staged_path.with_file_name(format!("{filename}.head.tmp")))
+}
+
+pub(super) async fn rewrite_subagent_history_boundary(
+ staged_path: &Path,
+ boundary: u64,
+) -> ThreadStoreResult<()> {
+ let rewritten_path = rewritten_staged_rollout_path(staged_path)?;
+ let permissions = tokio::fs::metadata(staged_path)
+ .await
+ .map_err(migration_error)?
+ .permissions();
+ let source = File::open(staged_path).await.map_err(migration_error)?;
+ let mut source = BufReader::new(source);
+ let mut head_bytes = Vec::new();
+ source
+ .read_until(b'\n', &mut head_bytes)
+ .await
+ .map_err(migration_error)?;
+ let mut head = serde_json::from_slice::<RolloutLine>(&head_bytes).map_err(migration_error)?;
+ let RolloutItem::SessionMeta(session_meta) = &mut head.item else {
+ return Err(migration_error(
+ "staged rollout head is not session metadata",
+ ));
+ };
+ session_meta.meta.subagent_history_start_ordinal = Some(boundary);
+
+ let rewritten = tokio::fs::OpenOptions::new()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(&rewritten_path)
+ .await
+ .map_err(migration_error)?;
+ rewritten
+ .set_permissions(permissions)
+ .await
+ .map_err(migration_error)?;
+ let mut rewritten = BufWriter::new(rewritten);
+ rewritten
+ .write_all(
+ serde_json::to_string(&head)
+ .map_err(migration_error)?
+ .as_bytes(),
+ )
+ .await
+ .map_err(migration_error)?;
+ rewritten.write_all(b"\n").await.map_err(migration_error)?;
+ tokio::io::copy(&mut source, &mut rewritten)
+ .await
+ .map_err(migration_error)?;
+ rewritten.flush().await.map_err(migration_error)?;
+ rewritten
+ .get_ref()
+ .sync_all()
+ .await
+ .map_err(migration_error)?;
+ drop(rewritten);
+ tokio::fs::rename(rewritten_path, staged_path)
+ .await
+ .map_err(migration_error)
+}
+
pub(super) async fn remove_file_if_present(path: &Path) -> ThreadStoreResult<()> {
if tokio::fs::try_exists(path).await.map_err(migration_error)? {
tokio::fs::remove_file(path)
diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback.rs
new file mode 100644
index 0000000000..428aecb45b
--- /dev/null
+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback.rs
@@ -0,0 +1,140 @@
+//! Defines which persisted history entries count as rollbackable user turns.
+//!
+//! Legacy `ThreadRolledBack { num_turns }` does not mean “drop the last N JSONL records.” A turn
+//! can include contextual user/developer messages, paired `ResponseItem` + `UserMessage` records,
+//! inter-agent communication, or compaction replacement history.
+//!
+//! This module mirrors the legacy cold-resume rollback boundary rules for persisted rollout shapes.
+//! It stays local to migration because core's live predicate also knows about dynamically
+//! registered contextual fragments that thread-store should not depend on.
+
+use codex_protocol::items::parse_hook_prompt_fragment;
+use codex_protocol::models::ContentItem;
+use codex_protocol::models::ResponseItem;
+use codex_protocol::protocol::InterAgentCommunication;
+
+/// Match the rollback boundaries used by legacy history reconstruction without
+/// treating every persisted lifecycle as a user turn. This intentionally stays
+/// local to the frozen migration adapter: the full core predicate also knows
+/// about dynamically registered contextual fragments that thread-store cannot
+/// depend on.
+pub(super) fn counts_as_boundary(response: &ResponseItem) -> bool {
+ if matches!(response, ResponseItem::AgentMessage { .. }) {
+ return true;
+ }
+ let ResponseItem::Message { role, content, .. } = response else {
+ return false;
+ };
+ (role == "user" && !is_known_contextual_user_message_content(content))
+ || (role == "assistant" && InterAgentCommunication::is_message_content(content))
+}
+
+pub(super) fn is_pre_turn_context_update(response: &ResponseItem) -> bool {
+ let ResponseItem::Message { role, content, .. } = response else {
+ return false;
+ };
+ (role == "user" && is_known_contextual_user_message_content(content))
+ || (role == "developer" && is_known_contextual_developer_message_content(content))
+}
+
+/// Apply the same response-history cut used by legacy cold resume to a persisted
+/// compaction checkpoint. The migration adapter uses a frozen contextual-fragment
+/// matcher because thread-store cannot depend on core's runtime fragment registry.
+pub(super) fn drop_last_n_user_turns(history: &mut Vec<ResponseItem>, num_turns: u32) {
+ if num_turns == 0 {
+ return;
+ }
+ let user_positions = history
+ .iter()
+ .enumerate()
+ .filter_map(|(index, item)| counts_as_boundary(item).then_some(index))
+ .collect::<Vec<_>>();
+ let Some(&first_turn_index) = user_positions.first() else {
+ return;
+ };
+ let count = usize::try_from(num_turns).unwrap_or(usize::MAX);
+ let mut cut_index = if count >= user_positions.len() {
+ first_turn_index
+ } else {
+ user_positions[user_positions.len() - count]
+ };
+ while cut_index > first_turn_index && is_pre_turn_context_update(&history[cut_index - 1]) {
+ cut_index -= 1;
+ }
+ history.truncate(cut_index);
+}
+
+fn is_known_contextual_user_message_content(content: &[ContentItem]) -> bool {
+ content.iter().any(|item| {
+ let ContentItem::InputText { text } = item else {
+ return false;
+ };
+ is_known_contextual_user_text(text)
+ })
+}
+
+fn is_known_contextual_developer_message_content(content: &[ContentItem]) -> bool {
+ content.iter().any(|item| {
+ let ContentItem::InputText { text } = item else {
+ return false;
+ };
+ let text = text.trim_start();
+ [
+ "<permissions instructions>",
+ "<model_switch>",
+ "<apps_instructions>",
+ "<collaboration_mode>",
+ "<multi_agent_mode>",
+ "<environments_instructions>",
+ "<git_attribution>",
+ "<plugins_instructions>",
+ "<realtime_conversation>",
+ "<skills_instructions>",
+ "<tools>",
+ "<personality_spec>",
+ "<token_budget>",
+ "<context_window>",
+ "<context_window_guidance>",
+ "<rollout_budget>",
+ ]
+ .iter()
+ .any(|prefix| {
+ text.get(..prefix.len())
+ .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
+ })
+ })
+}
+
+// Keep this frozen alongside the legacy migration adapter. Core's live predicate also knows
+// about dynamically registered fragments, but legacy rollouts only need these persisted shapes.
+fn is_known_contextual_user_text(text: &str) -> bool {
+ let text = text.trim();
+ parse_hook_prompt_fragment(text).is_some()
+ || [
+ ("# AGENTS.md instructions", "</INSTRUCTIONS>"),
+ ("<environment_context>", "</environment_context>"),
+ ("<skill>", "</skill>"),
+ ("<user_shell_command>", "</user_shell_command>"),
+ ("<turn_aborted>", "</turn_aborted>"),
+ ("<subagent_notification>", "</subagent_notification>"),
+ ("<recommended_plugins>", "</recommended_plugins>"),
+ ("<goal_context>", "</goal_context>"),
+ ]
+ .iter()
+ .any(|(start, end)| text.starts_with(start) && text.ends_with(end))
+ || (text.starts_with("<external_")
+ && text
+ .split_once('>')
+ .and_then(|(start, _)| start.strip_prefix("<external_"))
+ .is_some_and(|key| text.ends_with(&format!("</external_{key}>"))))
+ || (text.starts_with("<codex_internal_context")
+ && text.ends_with("</codex_internal_context>"))
+ || text.starts_with(
+ "Warning: The maximum number of unified exec processes you can keep open is",
+ )
+ || (text.starts_with("Warning: apply_patch was requested via ")
+ && text.ends_with("Use the apply_patch tool instead of exec_command."))
+ || text.starts_with(
+ "Warning: Your account was flagged for potentially high-risk cyber activity",
+ )
+}
diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
new file mode 100644
index 0000000000..d28c1c09a2
--- /dev/null
+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
@@ -0,0 +1,380 @@
+//! Decides which legacy records remain visible after historical rollback.
+//!
+//! Legacy rollback removes logical instruction turns, not a physical suffix of the rollout file.
+//! Most records happen to be ordered that way, but late completion events can target an older
+//! surviving turn after a newer turn has started. This planner keeps compact per-record ownership
+//! metadata for SQLite visibility, then combines it with `rollback_replay`'s cold-resume answer
+//! before the writer makes its second streaming pass.
+
+use std::collections::HashMap;
+
+use codex_protocol::models::ContentItem;
+use codex_protocol::models::ResponseItem;
+use codex_protocol::protocol::CompactedItem;
+use codex_protocol::protocol::EventMsg;
+use codex_protocol::protocol::RolloutItem;
+use codex_protocol::protocol::RolloutLine;
+use codex_protocol::protocol::UserMessageEvent;
+
+use super::migration_error;
+use super::rollback;
+use super::rollback_replay::ModelReplayPlanner;
+use crate::ThreadStoreResult;
+
+#[derive(Clone)]
+struct CompactionFrame {
+ record_index: usize,
+ boundary_depth: usize,
+ owner: Option<usize>,
+ item: CompactedItem,
+}
+
+#[derive(Clone)]
+struct PendingUserResponse {
+ boundary: usize,
+ content: Vec<ContentItem>,
+}
+
+/// Compact plan keyed by parsed source-record index.
+pub(super) struct RollbackPlan {
+ record_boundaries: Vec<Option<usize>>,
+ boundary_alive: Vec<bool>,
+ compacted_items: HashMap<usize, CompactedItem>,
+}
+
+impl RollbackPlan {
+ pub(super) fn record_count(&self) -> usize {
+ self.record_boundaries.len()
+ }
+
+ pub(super) fn apply(
+ &self,
+ record_index: usize,
+ mut line: RolloutLine,
+ ) -> ThreadStoreResult<Option<RolloutLine>> {
+ let boundary = self
+ .record_boundaries
+ .get(record_index)
+ .ok_or_else(|| migration_error("rollback plan is shorter than source replay"))?;
+ if matches!(
+ &line.item,
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))
+ ) {
+ return Ok(None);
+ }
+ // A rolled-back turn can still own the empty checkpoint that keeps cold resume from
+ // replaying older history.
+ if let Some(compacted) = self.compacted_items.get(&record_index) {
+ line.item = RolloutItem::Compacted(compacted.clone());
+ return Ok(Some(line));
+ }
+ if boundary.is_some_and(|boundary| !self.boundary_alive[boundary]) {
+ return Ok(None);
+ }
+ Ok(Some(line))
+ }
+}
+
+/// Streaming builder for RollbackPlan.
+pub(super) struct RollbackPlanner {
+ record_boundaries: Vec<Option<usize>>,
+ boundary_alive: Vec<bool>,
+ boundary_stack: Vec<usize>,
+ active_turn_id: Option<String>,
+ pending_turn_records: Vec<usize>,
+ pending_context_records: Vec<usize>,
+ pending_user_response: Option<PendingUserResponse>,
+ pending_delivery_boundary: Option<usize>,
+ turn_boundaries: HashMap<String, usize>,
+ compactions: Vec<CompactionFrame>,
+ model_replay: ModelReplayPlanner,
+}
+
+impl RollbackPlanner {
+ pub(super) fn new() -> Self {
+ Self {
+ record_boundaries: Vec::new(),
+ boundary_alive: Vec::new(),
+ boundary_stack: Vec::new(),
+ active_turn_id: None,
+ pending_turn_records: Vec::new(),
+ pending_context_records: Vec::new(),
+ pending_user_response: None,
+ pending_delivery_boundary: None,
+ turn_boundaries: HashMap::new(),
+ compactions: Vec::new(),
+ model_replay: ModelReplayPlanner::new(),
+ }
+ }
+
+ pub(super) fn observe(&mut self, line: &RolloutLine) -> ThreadStoreResult<()> {
+ let index = self.record_boundaries.len();
+ self.model_replay.observe(index, &line.item);
+ self.record_boundaries
+ .push(self.boundary_stack.last().copied());
+ let paired_user_boundary = match (&self.pending_user_response, &line.item) {
+ (Some(pending), RolloutItem::EventMsg(EventMsg::UserMessage(event)))
+ if user_response_matches_event(&pending.content, event) =>
+ {
+ Some(pending.boundary)
+ }
+ _ => None,
+ };
+ let paired_delivery_boundary = match (&self.pending_delivery_boundary, &line.item) {
+ (Some(boundary), RolloutItem::ResponseItem(ResponseItem::AgentMessage { .. })) => {
+ Some(*boundary)
+ }
+ _ => None,
+ };
+ self.pending_user_response = None;
+ self.pending_delivery_boundary = None;
+
+ match &line.item {
+ RolloutItem::SessionMeta(_) => self.record_boundaries[index] = None,
+ RolloutItem::ResponseItem(response) => {
+ if let Some(boundary) = paired_delivery_boundary {
+ self.record_boundaries[index] = Some(boundary);
+ } else if rollback::counts_as_boundary(response) {
+ let boundary = self.start_boundary(index);
+ if let ResponseItem::Message { role, content, .. } = response
+ && role == "user"
+ {
+ self.pending_user_response = Some(PendingUserResponse {
+ boundary,
+ content: content.clone(),
+ });
+ }
+ } else if rollback::is_pre_turn_context_update(response) {
+ // Until another user boundary arrives, this is trailing context for the
+ // previous turn. Keep that fallback owner so rollback drops it when there is
+ // no later turn to attach it to.
+ self.pending_context_records.push(index);
+ }
+ }
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {
+ self.apply_rollback(rollback.num_turns)?;
+ }
+ RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
+ self.active_turn_id = Some(event.turn_id.clone());
+ self.pending_turn_records.clear();
+ self.pending_turn_records.push(index);
+ }
+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
+ self.assign_targeted_record(index, Some(event.turn_id.as_str()));
+ if self.active_turn_id.as_deref() == Some(event.turn_id.as_str()) {
+ self.active_turn_id = None;
+ self.pending_turn_records.clear();
+ }
+ }
+ RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {
+ self.assign_targeted_record(index, event.turn_id.as_deref());
+ if event
+ .turn_id
+ .as_deref()
+ .is_some_and(|turn_id| self.active_turn_id.as_deref() == Some(turn_id))
+ {
+ self.active_turn_id = None;
+ self.pending_turn_records.clear();
+ }
+ }
+ RolloutItem::EventMsg(EventMsg::UserMessage(_)) => {
+ let boundary = paired_user_boundary.unwrap_or_else(|| self.start_boundary(index));
+ self.record_boundaries[index] = Some(boundary);
+ }
+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => {
+ self.assign_targeted_record(index, Some(event.turn_id.as_str()));
+ }
+ RolloutItem::EventMsg(event) => {
+ self.assign_targeted_record(index, explicit_event_turn_id(event));
+ }
+ RolloutItem::InterAgentCommunication(_) => {
+ self.start_boundary(index);
+ }
+ RolloutItem::InterAgentCommunicationMetadata { .. } => {
+ let boundary = self.start_boundary(index);
+ self.pending_delivery_boundary = Some(boundary);
+ }
+ RolloutItem::Compacted(item) => {
+ let owner = self
+ .active_turn_id
+ .as_deref()
+ .and_then(|turn_id| self.turn_boundaries.get(turn_id).copied());
+ self.record_boundaries[index] = owner;
+ self.compactions.push(CompactionFrame {
+ record_index: index,
+ boundary_depth: self.boundary_stack.len(),
+ owner,
+ item: item.clone(),
+ });
+ }
+ RolloutItem::TurnContext(_) => {
+ if self.active_turn_id.is_some()
+ && self
+ .active_turn_id
+ .as_deref()
+ .is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))
+ {
+ self.pending_turn_records.push(index);
+ }
+ }
+ RolloutItem::WorldState(_) => {}
+ }
+
+ Ok(())
+ }
+
+ pub(super) fn finish(self) -> RollbackPlan {
+ let RollbackPlanner {
+ record_boundaries,
+ boundary_alive,
+ compactions,
+ model_replay,
+ ..
+ } = self;
+ let replay_anchor = model_replay.finish().empty_replacement_history_compaction;
+ let compacted_items = compactions
+ .into_iter()
+ .filter_map(|mut frame| {
+ if Some(frame.record_index) == replay_anchor {
+ frame.item.replacement_history = Some(Vec::new());
+ return Some((frame.record_index, frame.item));
+ }
+ frame
+ .owner
+ .is_none_or(|boundary| boundary_alive[boundary])
+ .then_some((frame.record_index, frame.item))
+ })
+ .collect::<HashMap<_, _>>();
+ RollbackPlan {
+ record_boundaries,
+ boundary_alive,
+ compacted_items,
+ }
+ }
+
+ fn start_boundary(&mut self, index: usize) -> usize {
+ let boundary = self.boundary_alive.len();
+ self.boundary_alive.push(true);
+ let had_prior_boundary = !self.boundary_stack.is_empty();
+ if had_prior_boundary {
+ for pending_index in self.pending_context_records.drain(..) {
+ self.record_boundaries[pending_index] = Some(boundary);
+ }
+ } else {
+ self.pending_context_records.clear();
+ }
+ for pending_index in self.pending_turn_records.drain(..) {
+ self.record_boundaries[pending_index] = Some(boundary);
+ }
+ self.record_boundaries[index] = Some(boundary);
+ self.boundary_stack.push(boundary);
+ self.bind_active_turn(boundary);
+ boundary
+ }
+
+ fn bind_active_turn(&mut self, boundary: usize) {
+ if let Some(turn_id) = self.active_turn_id.as_ref() {
+ self.turn_boundaries.insert(turn_id.clone(), boundary);
+ }
+ }
+
+ fn assign_targeted_record(&mut self, index: usize, turn_id: Option<&str>) {
+ if let Some(boundary) = turn_id.and_then(|turn_id| self.turn_boundaries.get(turn_id)) {
+ self.record_boundaries[index] = Some(*boundary);
+ } else if self.active_turn_id.is_some()
+ && self
+ .active_turn_id
+ .as_deref()
+ .is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))
+ {
+ self.pending_turn_records.push(index);
+ }
+ }
+
+ fn apply_rollback(&mut self, num_turns: u32) -> ThreadStoreResult<()> {
+ let count = usize::try_from(num_turns).unwrap_or(usize::MAX);
+ if count == 0 {
+ return Ok(());
+ }
+ let depth_before = self.boundary_stack.len();
+ for _ in 0..count {
+ let Some(boundary) = self.boundary_stack.pop() else {
+ break;
+ };
+ self.boundary_alive[boundary] = false;
+ }
+ let compaction_index = self.compactions.iter().rposition(|frame| {
+ frame
+ .owner
+ .is_none_or(|boundary| self.boundary_alive[boundary])
+ });
+ if let Some(compaction_index) = compaction_index {
+ let frame = &mut self.compactions[compaction_index];
+ let post_compaction_turns = depth_before.saturating_sub(frame.boundary_depth);
+ let remaining = count.saturating_sub(post_compaction_turns);
+ if remaining > 0 {
+ let replacement_history =
+ frame.item.replacement_history.as_mut().ok_or_else(|| {
+ migration_error(
+ "legacy rollback crosses a compaction without replacement history",
+ )
+ })?;
+ rollback::drop_last_n_user_turns(
+ replacement_history,
+ u32::try_from(remaining).unwrap_or(u32::MAX),
+ );
+ }
+ }
+ self.active_turn_id = None;
+ self.pending_turn_records.clear();
+ self.pending_context_records.clear();
+ self.pending_user_response = None;
+ self.pending_delivery_boundary = None;
+ Ok(())
+ }
+}
+
+fn explicit_event_turn_id(event: &EventMsg) -> Option<&str> {
+ match event {
+ EventMsg::ExecCommandEnd(event) => Some(event.turn_id.as_str()),
+ EventMsg::PatchApplyEnd(event) => Some(event.turn_id.as_str()),
+ EventMsg::DynamicToolCallResponse(event) => Some(event.turn_id.as_str()),
+ EventMsg::EnteredReviewMode(event) => event.turn_id.as_deref(),
+ EventMsg::ExitedReviewMode(event) => event.turn_id.as_deref(),
+ _ => None,
+ }
+ .filter(|turn_id| !turn_id.is_empty())
+}
+
+fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent) -> bool {
+ let mut text = String::new();
+ let mut images = Vec::new();
+ let mut audio = Vec::new();
+ for item in content {
+ match item {
+ ContentItem::InputText { text: item_text } => text.push_str(item_text),
+ ContentItem::InputImage { image_url, .. } => images.push(image_url.as_str()),
+ ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
+ ContentItem::OutputText { .. } => return false,
+ }
+ }
+ text == event.message
+ && images
+ == event
+ .images
+ .as_deref()
+ .unwrap_or_default()
+ .iter()
+ .map(String::as_str)
+ .collect::<Vec<_>>()
+ && audio
+ == event
+ .audio
+ .as_deref()
+ .unwrap_or_default()
+ .iter()
+ .map(String::as_str)
+ .collect::<Vec<_>>()
+ && event.local_images.is_empty()
+ && event.local_audio.is_empty()
+}
diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs
new file mode 100644
index 0000000000..8d0cb30fd9
--- /dev/null
+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs
@@ -0,0 +1,191 @@
+//! Mirrors the part of legacy cold resume that chooses the model-history checkpoint.
+//!
+//! Legacy rollback has two jobs. The forward history reducer decides which turns remain visible,
+//! while cold resume scans backward to find the newest surviving compaction base. Migration needs
+//! both answers because removing rollback markers must not change what the next resume sends to
+//! the model.
+
+use codex_protocol::protocol::EventMsg;
+use codex_protocol::protocol::RolloutItem;
+
+use super::rollback;
+
+enum ReplayRecord {
+ Compacted {
+ record_index: usize,
+ has_replacement_history: bool,
+ },
+ Rollback(u32),
+ TurnStarted(String),
+ TurnComplete(String),
+ TurnAborted(Option<String>),
+ TurnContext(Option<String>),
+ UserBoundary,
+}
+
+#[derive(Default)]
+struct ReverseSegment {
+ turn_id: Option<String>,
+ counts_as_user_turn: bool,
+ compaction_index: Option<usize>,
+}
+
+/// Rewrite needed when legacy reverse replay skips a compaction segment but still resumes from
+/// the suffix after that checkpoint.
+pub(super) struct ModelReplayPlan {
+ pub(super) empty_replacement_history_compaction: Option<usize>,
+}
+
+pub(super) struct ModelReplayPlanner {
+ records: Vec<ReplayRecord>,
+}
+
+impl ModelReplayPlanner {
+ pub(super) fn new() -> Self {
+ Self {
+ records: Vec::new(),
+ }
+ }
+
+ pub(super) fn observe(&mut self, record_index: usize, item: &RolloutItem) {
+ let record = match item {
+ RolloutItem::Compacted(compacted) => ReplayRecord::Compacted {
+ record_index,
+ has_replacement_history: compacted.replacement_history.is_some(),
+ },
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {
+ ReplayRecord::Rollback(rollback.num_turns)
+ }
+ RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
+ ReplayRecord::TurnStarted(event.turn_id.clone())
+ }
+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
+ ReplayRecord::TurnComplete(event.turn_id.clone())
+ }
+ RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {
+ ReplayRecord::TurnAborted(event.turn_id.clone())
+ }
+ RolloutItem::TurnContext(context) => ReplayRecord::TurnContext(context.turn_id.clone()),
+ RolloutItem::EventMsg(EventMsg::UserMessage(_))
+ | RolloutItem::InterAgentCommunication(_) => ReplayRecord::UserBoundary,
+ RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(response) => {
+ ReplayRecord::UserBoundary
+ }
+ RolloutItem::SessionMeta(_)
+ | RolloutItem::ResponseItem(_)
+ | RolloutItem::EventMsg(_)
+ | RolloutItem::InterAgentCommunicationMetadata { .. }
+ | RolloutItem::WorldState(_) => return,
+ };
+ self.records.push(record);
+ }
+
+ pub(super) fn finish(self) -> ModelReplayPlan {
+ let mut selected_compaction = None;
+ let mut suffix_start = None;
+ let mut pending_rollback_turns = 0usize;
+ let mut active_segment: Option<ReverseSegment> = None;
+
+ for record in self.records.iter().rev() {
+ match record {
+ ReplayRecord::Compacted {
+ record_index,
+ has_replacement_history,
+ } => {
+ let segment = active_segment.get_or_insert_with(ReverseSegment::default);
+ if segment.compaction_index.is_none() && *has_replacement_history {
+ segment.compaction_index = Some(*record_index);
+ suffix_start = record_index.checked_add(1);
+ }
+ }
+ ReplayRecord::Rollback(num_turns) => {
+ pending_rollback_turns = pending_rollback_turns
+ .saturating_add(usize::try_from(*num_turns).unwrap_or(usize::MAX));
+ }
+ ReplayRecord::TurnComplete(turn_id) => {
+ let segment = active_segment.get_or_insert_with(ReverseSegment::default);
+ if segment.turn_id.is_none() {
+ segment.turn_id = Some(turn_id.clone());
+ }
+ }
+ ReplayRecord::TurnAborted(turn_id) => {
+ if let Some(segment) = active_segment.as_mut() {
+ if segment.turn_id.is_none() {
+ segment.turn_id = turn_id.clone();
+ }
+ } else if turn_id.is_some() {
+ active_segment = Some(ReverseSegment {
+ turn_id: turn_id.clone(),
+ ..Default::default()
+ });
+ }
+ }
+ ReplayRecord::TurnContext(turn_id) => {
+ let segment = active_segment.get_or_insert_with(ReverseSegment::default);
+ if segment.turn_id.is_none() {
+ segment.turn_id = turn_id.clone();
+ }
+ }
+ ReplayRecord::UserBoundary => {
+ active_segment
+ .get_or_insert_with(ReverseSegment::default)
+ .counts_as_user_turn = true;
+ }
+ ReplayRecord::TurnStarted(turn_id) => {
+ if active_segment.as_ref().is_some_and(|segment| {
+ turn_ids_are_compatible(segment.turn_id.as_deref(), Some(turn_id.as_str()))
+ }) && let Some(segment) = active_segment.take()
+ {
+ finalize_segment(
+ segment,
+ &mut selected_compaction,
+ &mut pending_rollback_turns,
+ );
+ }
+ }
+ }
+ }
+
+ if let Some(segment) = active_segment {
+ finalize_segment(
+ segment,
+ &mut selected_compaction,
+ &mut pending_rollback_turns,
+ );
+ }
+
+ // If reverse replay saw a replacement-history checkpoint but rolled back its segment,
+ // it resumes from that checkpoint's suffix without using the checkpoint history. Keep an
+ // empty checkpoint in the paginated file so older SQLite-visible records do not leak back
+ // into model context after the rollback markers are gone.
+ let empty_replacement_history_compaction = selected_compaction
+ .is_none()
+ .then_some(suffix_start)
+ .flatten()
+ .and_then(|start| start.checked_sub(1));
+ ModelReplayPlan {
+ empty_replacement_history_compaction,
+ }
+ }
+}
+
+fn finalize_segment(
+ segment: ReverseSegment,
+ selected_compaction: &mut Option<usize>,
+ pending_rollback_turns: &mut usize,
+) {
+ if *pending_rollback_turns > 0 {
+ if segment.counts_as_user_turn {
+ *pending_rollback_turns -= 1;
+ }
+ return;
+ }
+ if selected_compaction.is_none() {
+ *selected_compaction = segment.compaction_index;
+ }
+}
+
+fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool {
+ active_turn_id
+ .is_none_or(|turn_id| item_turn_id.is_none_or(|item_turn_id| item_turn_id == turn_id))
+}
diff --git a/codex-rs/thread-store/src/local/rollout_migration/subagent.rs b/codex-rs/thread-store/src/local/rollout_migration/subagent.rs
new file mode 100644
index 0000000000..54cfc75909
--- /dev/null
+++ b/codex-rs/thread-store/src/local/rollout_migration/subagent.rs
@@ -0,0 +1,59 @@
+//! Selects a smaller resume context for legacy subagent rollouts.
+//!
+//! Legacy subagents copied the parent's full rollout into every child rollout. Migrating those
+//! files verbatim can preserve gigabytes of duplicated history even though a resumed subagent only
+//! needs its latest bounded model context.
+//!
+//! This module reverse-scans the legacy rollout for a safe model-context checkpoint and returns the
+//! smallest replay needed to resume from there. Malformed reverse-scanned records are skipped. If
+//! we cannot prove that a bounded replay is safe, we return `None` and let the caller fall back to
+//! full tolerant replay.
+
+use std::fs::File;
+use std::path::PathBuf;
+
+use codex_protocol::protocol::RolloutItem;
+use codex_protocol::protocol::SessionMetaLine;
+use codex_rollout::ModelContextScan;
+use codex_rollout::ModelContextScanProgress;
+use codex_rollout::ReverseJsonlScanner;
+use codex_rollout::ScanOutcome;
+use serde_json::Value;
+
+use super::line_parser;
+use super::migration_error;
+use crate::ThreadStoreResult;
+
+/// Returns a bounded replay when the legacy child has enough durable context to reconstruct from
+/// a suffix. Without that proof, the caller streams the complete legacy replay instead.
+pub(super) async fn select_bounded_context(
+ rollout_path: PathBuf,
+ session_meta: SessionMetaLine,
+) -> ThreadStoreResult<Option<Vec<RolloutItem>>> {
+ tokio::task::spawn_blocking(move || {
+ let file = File::open(rollout_path).map_err(migration_error)?;
+ let mut scanner = ReverseJsonlScanner::new(file)
+ .map_err(migration_error)?
+ .with_max_record_bytes(super::MAX_ROLLOUT_LINE_BYTES);
+ let mut scan = ModelContextScan::default();
+
+ while let Some(outcome) = scanner.scan_next::<Value>().map_err(migration_error)? {
+ let value = match outcome {
+ ScanOutcome::Parsed(value) => value,
+ ScanOutcome::Rejected(_) => continue,
+ };
+ let Ok(Some(line)) = line_parser::parse_legacy_rollout_value(value) else {
+ continue;
+ };
+ if scan.push(line.item) == ModelContextScanProgress::Complete {
+ let mut items = scan.finish(session_meta);
+ items.retain(|item| !matches!(item, RolloutItem::SessionMeta(_)));
+ return Ok(Some(items));
+ }
+ }
+
+ Ok(None)
+ })
+ .await
+ .map_err(migration_error)?
+}
diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rs
index dcdae5750e..7761ca382b 100644
--- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs
@@ -6,21 +6,36 @@ use std::path::PathBuf;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
+use codex_app_server_protocol::build_turns_from_rollout_items;
+use codex_protocol::AgentPath;
use codex_protocol::ThreadId;
+use codex_protocol::config_types::ReasoningSummary;
+use codex_protocol::items::ReasoningItem;
+use codex_protocol::items::TurnItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AgentMessageEvent;
+use codex_protocol::protocol::AskForApproval;
+use codex_protocol::protocol::CompactedItem;
use codex_protocol::protocol::EventMsg;
+use codex_protocol::protocol::InterAgentCommunication;
+use codex_protocol::protocol::ItemCompletedEvent;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
+use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadHistoryMode;
+use codex_protocol::protocol::ThreadRolledBackEvent;
+use codex_protocol::protocol::TurnCompleteEvent;
+use codex_protocol::protocol::TurnContextItem;
+use codex_protocol::protocol::TurnStartedEvent;
use codex_protocol::protocol::UserMessageEvent;
use codex_rollout::RolloutConfig;
use pretty_assertions::assert_eq;
+use serde_json::json;
use tempfile::TempDir;
use super::LocalThreadStore;
@@ -32,9 +47,15 @@ use super::decompress_rollout_to_path;
use super::migration_journal_path;
use super::thread_history;
use super::write_migration_journal;
+use crate::ItemSortKey;
+use crate::ListItemsParams;
use crate::ListTurnsParams;
+use crate::LoadThreadHistoryParams;
use crate::SortDirection;
use crate::StoredTurnItemsView;
+use crate::ThreadStore;
+use crate::ThreadStoreError;
+use crate::TurnPage;
use crate::local::test_support::test_config;
const TIMESTAMP: &str = "2025-01-03T12:00:00Z";
@@ -92,6 +113,26 @@ fn write_rollout_with_fork(
path
}
+fn move_to_archived(home: &Path, path: PathBuf) -> PathBuf {
+ let directory = home.join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR);
+ fs::create_dir_all(&directory).expect("create archived rollout directory");
+ let archived_path = directory.join(path.file_name().expect("rollout filename"));
+ fs::rename(path, &archived_path).expect("archive rollout");
+ archived_path
+}
+
+fn compress_rollout(path: &Path) -> PathBuf {
+ let compressed_path = path.with_extension("jsonl.zst");
+ let compressed = zstd::stream::encode_all(
+ fs::File::open(path).expect("open rollout"),
+ /*level*/ 3,
+ )
+ .expect("compress rollout");
+ fs::write(&compressed_path, compressed).expect("write compressed rollout");
+ fs::remove_file(path).expect("remove plain rollout");
+ compressed_path
+}
+
fn user_message(text: &str) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: text.to_string(),
@@ -107,6 +148,88 @@ fn agent_message(text: &str) -> RolloutItem {
}))
}
+fn input_response_message(role: &str, text: &str) -> ResponseItem {
+ ResponseItem::Message {
+ id: None,
+ role: role.to_string(),
+ content: vec![ContentItem::InputText {
+ text: text.to_string(),
+ }],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ }
+}
+
+fn exec_completion(turn_id: &str, call_id: &str) -> RolloutItem {
+ serde_json::from_value(json!({
+ "type": "event_msg",
+ "payload": {
+ "type": "exec_command_end",
+ "call_id": call_id,
+ "turn_id": turn_id,
+ "command": ["echo", "ok"],
+ "cwd": "file:///tmp",
+ "parsed_cmd": [],
+ "source": "agent",
+ "stdout": "ok",
+ "stderr": "",
+ "aggregated_output": "ok",
+ "exit_code": 0,
+ "duration": {"secs": 0, "nanos": 0},
+ "formatted_output": "ok",
+ "status": "completed"
+ }
+ }))
+ .expect("build legacy exec completion")
+}
+
+fn item_completed(turn_id: &str, item_id: &str) -> RolloutItem {
+ RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
+ thread_id: ThreadId::new(),
+ turn_id: turn_id.to_string(),
+ item: TurnItem::Reasoning(ReasoningItem {
+ id: item_id.to_string(),
+ summary_text: vec!["summary".to_string()],
+ raw_content: Vec::new(),
+ }),
+ started_at_ms: None,
+ completed_at_ms: 1_735_905_601_000,
+ }))
+}
+
+fn started(turn_id: &str) -> RolloutItem {
+ RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
+ turn_id: turn_id.to_string(),
+ trace_id: None,
+ started_at: Some(1_735_905_600),
+ model_context_window: None,
+ collaboration_mode_kind: Default::default(),
+ }))
+}
+
+fn compacted(replacement_history: Vec<ResponseItem>) -> RolloutItem {
+ RolloutItem::Compacted(CompactedItem {
+ message: "checkpoint".to_string(),
+ replacement_history: Some(replacement_history),
+ window_number: Some(1),
+ first_window_id: None,
+ previous_window_id: None,
+ window_id: None,
+ })
+}
+
+fn completed(turn_id: &str) -> RolloutItem {
+ RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
+ turn_id: turn_id.to_string(),
+ last_agent_message: None,
+ error: None,
+ started_at: Some(1_735_905_600),
+ completed_at: Some(1_735_905_601),
+ duration_ms: Some(1_000),
+ time_to_first_token_ms: None,
+ }))
+}
+
fn read_rollout(path: &Path) -> Vec<RolloutLine> {
fs::read_to_string(path)
.expect("read migrated rollout")
@@ -115,15 +238,6 @@ fn read_rollout(path: &Path) -> Vec<RolloutLine> {
.collect()
}
-fn compress_rollout(path: PathBuf) -> PathBuf {
- let compressed_path = path.with_extension("jsonl.zst");
- let input = fs::File::open(&path).expect("open rollout for compression");
- let compressed = zstd::stream::encode_all(input, /*level*/ 3).expect("compress rollout");
- fs::write(&compressed_path, compressed).expect("write compressed rollout");
- fs::remove_file(path).expect("remove plain rollout");
- compressed_path
-}
-
fn apply_options() -> RolloutMigrationOptions {
RolloutMigrationOptions {
mode: RolloutMigrationMode::Apply,
@@ -147,6 +261,20 @@ async fn indexed_store(home: &Path) -> LocalThreadStore {
LocalThreadStore::new(config, Some(state_db))
}
+async fn list_active_summary_turns(store: &LocalThreadStore, thread_id: ThreadId) -> TurnPage {
+ store
+ .list_turns(ListTurnsParams {
+ thread_id,
+ include_archived: false,
+ cursor: None,
+ page_size: 10,
+ sort_direction: SortDirection::Asc,
+ items_view: StoredTurnItemsView::Summary,
+ })
+ .await
+ .expect("read projected turns")
+}
+
#[tokio::test]
async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
let home = TempDir::new().expect("create Codex home");
@@ -189,6 +317,174 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
2
);
+ let turns = list_active_summary_turns(&store, thread_id).await;
+ assert_eq!(turns.turns.len(), 1);
+ assert_eq!(turns.turns[0].items.len(), 2);
+
+ let bytes = fs::read(&path).expect("read first migration");
+ let second = store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("rerun migration");
+ assert_eq!(
+ second.outcomes[0].status,
+ RolloutMigrationStatus::AlreadyPaginated
+ );
+ assert_eq!(fs::read(&path).expect("read idempotent rollout"), bytes);
+}
+
+#[tokio::test]
+async fn migration_projects_explicit_and_implicit_legacy_completed_items() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let exec = exec_completion("explicit", "call-1");
+ let reasoning = serde_json::from_value(json!({
+ "type": "event_msg",
+ "payload": {"type": "agent_reasoning", "text": "summary"}
+ }))
+ .expect("build legacy reasoning summary");
+ let raw_reasoning = serde_json::from_value(json!({
+ "type": "event_msg",
+ "payload": {"type": "agent_reasoning_raw_content", "text": "raw"}
+ }))
+ .expect("build legacy reasoning content");
+ write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("explicit"),
+ exec,
+ completed("explicit"),
+ reasoning,
+ raw_reasoning,
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate legacy completion events");
+
+ let turns = list_active_summary_turns(&store, thread_id).await;
+ assert_eq!(turns.turns.len(), 2);
+ assert_eq!(turns.turns[0].turn_id, "explicit");
+ let items = store
+ .list_items(ListItemsParams {
+ thread_id,
+ turn_id: None,
+ include_archived: false,
+ cursor: None,
+ page_size: 10,
+ sort_direction: SortDirection::Asc,
+ sort_key: ItemSortKey::CreatedAtOrdinal,
+ after_updated_at_ordinal: None,
+ })
+ .await
+ .expect("read projected items");
+ assert_eq!(items.items.len(), 2);
+ assert_eq!(items.items[0].turn_id, "explicit");
+ assert_eq!(items.items[1].turn_id, turns.turns[1].turn_id);
+ let command: serde_json::Value =
+ serde_json::from_slice(&items.items[0].item_json).expect("parse projected command");
+ let reasoning: serde_json::Value =
+ serde_json::from_slice(&items.items[1].item_json).expect("parse projected reasoning");
+ assert_eq!(command["type"], "commandExecution");
+ assert_eq!(reasoning["type"], "reasoning");
+ assert_eq!(reasoning["summary"], json!(["summary"]));
+ assert_eq!(reasoning["content"], json!(["raw"]));
+}
+
+#[tokio::test]
+async fn migration_keeps_late_completions_in_their_original_turn() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("old"),
+ user_message("old question"),
+ started("current"),
+ user_message("current question"),
+ exec_completion("old", "call-old"),
+ agent_message("current answer"),
+ completed("current"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate late completion");
+
+ let started_turn_ids = read_rollout(&path)
+ .into_iter()
+ .filter_map(|line| match line.item {
+ RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => Some(event.turn_id),
+ _ => None,
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(started_turn_ids, vec!["old", "current"]);
+
+ let items = store
+ .list_items(ListItemsParams {
+ thread_id,
+ turn_id: None,
+ include_archived: false,
+ cursor: None,
+ page_size: 10,
+ sort_direction: SortDirection::Asc,
+ sort_key: ItemSortKey::CreatedAtOrdinal,
+ after_updated_at_ordinal: None,
+ })
+ .await
+ .expect("read projected late completion");
+ let item_turn_ids = items
+ .items
+ .iter()
+ .map(|item| item.turn_id.as_str())
+ .collect::<Vec<_>>();
+ assert_eq!(item_turn_ids, vec!["old", "current", "old", "current"]);
+}
+
+#[tokio::test]
+async fn migration_hoists_delayed_session_meta_before_paginated_history() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(home.path(), thread_id, SessionSource::Cli, Vec::new());
+ let existing = fs::read_to_string(&path).expect("read legacy rollout");
+ let pre_header = RolloutLine {
+ timestamp: TIMESTAMP.to_string(),
+ ordinal: None,
+ item: user_message("before metadata"),
+ };
+ fs::write(
+ &path,
+ format!(
+ "{}\n{existing}",
+ serde_json::to_string(&pre_header).expect("serialize pre-header record")
+ ),
+ )
+ .expect("write pre-header rollout");
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate delayed session metadata");
+
+ let lines = read_rollout(&path);
+ assert_eq!(lines[0].ordinal, Some(0));
+ assert!(matches!(
+ &lines[0].item,
+ RolloutItem::SessionMeta(metadata)
+ if metadata.meta.id == thread_id
+ && metadata.meta.history_mode == ThreadHistoryMode::Paginated
+ ));
let turns = store
.list_turns(ListTurnsParams {
thread_id,
@@ -196,14 +492,12 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
cursor: None,
page_size: 10,
sort_direction: SortDirection::Asc,
- items_view: StoredTurnItemsView::Summary,
+ items_view: StoredTurnItemsView::NotLoaded,
})
.await
- .expect("read projected turns");
+ .expect("read migrated turns");
assert_eq!(turns.turns.len(), 1);
- assert_eq!(turns.turns[0].items.len(), 2);
- let bytes = fs::read(&path).expect("read first migration");
let second = store
.migrate_rollouts(apply_options())
.await
@@ -212,43 +506,918 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
second.outcomes[0].status,
RolloutMigrationStatus::AlreadyPaginated
);
- assert_eq!(fs::read(&path).expect("read idempotent rollout"), bytes);
}
#[tokio::test]
-async fn migration_preserves_valid_final_record_without_newline() {
+async fn migration_preserves_valid_final_record_without_newline() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![user_message("question")],
+ );
+ let final_line = RolloutLine {
+ timestamp: TIMESTAMP.to_string(),
+ ordinal: None,
+ item: agent_message("answer"),
+ };
+ fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .expect("open legacy rollout")
+ .write_all(
+ serde_json::to_string(&final_line)
+ .expect("serialize final record")
+ .as_bytes(),
+ )
+ .expect("append final record");
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate legacy rollout");
+
+ assert_eq!(
+ read_rollout(&path)
+ .iter()
+ .filter(|line| matches!(line.item, RolloutItem::EventMsg(EventMsg::ItemCompleted(_))))
+ .count(),
+ 2
+ );
+}
+
+#[tokio::test]
+async fn migration_applies_historical_rollbacks_before_sqlite_projection() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("keep"),
+ user_message("keep question"),
+ agent_message("keep answer"),
+ completed("keep"),
+ started("remove"),
+ user_message("remove question"),
+ agent_message("remove answer"),
+ completed("remove"),
+ started("shell"),
+ completed("shell"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ started("replacement"),
+ user_message("replacement question"),
+ agent_message("replacement answer"),
+ completed("replacement"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ let report = store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate rolled-back thread");
+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
+
+ let lines = read_rollout(&path);
+ assert!(!lines.iter().any(|line| matches!(
+ line.item,
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))
+ )));
+ let turns = list_active_summary_turns(&store, thread_id).await;
+ assert_eq!(
+ turns
+ .turns
+ .iter()
+ .map(|turn| turn.turn_id.as_str())
+ .collect::<Vec<_>>(),
+ vec!["keep", "replacement"]
+ );
+}
+
+#[tokio::test]
+async fn migration_rolls_back_response_and_inter_agent_user_boundaries() {
+ let home = TempDir::new().expect("create Codex home");
+ let response_thread_id = ThreadId::new();
+ let response_path = write_rollout(
+ home.path(),
+ response_thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::ResponseItem(input_response_message("user", "remove response boundary")),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let communication_thread_id = ThreadId::new();
+ let communication_path = write_rollout(
+ home.path(),
+ communication_thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::InterAgentCommunication(InterAgentCommunication::new(
+ AgentPath::root(),
+ AgentPath::root().join("worker").expect("worker path"),
+ Vec::new(),
+ "remove communication boundary".to_string(),
+ /*trigger_turn*/ true,
+ )),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let contextual_thread_id = ThreadId::new();
+ let contextual_path = write_rollout(
+ home.path(),
+ contextual_thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::ResponseItem(input_response_message("user", "keep first boundary")),
+ RolloutItem::ResponseItem(input_response_message(
+ "developer",
+ "<permissions instructions>context only</permissions instructions>",
+ )),
+ RolloutItem::ResponseItem(input_response_message(
+ "user",
+ "<environment_context>context only</environment_context>",
+ )),
+ RolloutItem::ResponseItem(input_response_message("user", "remove real user boundary")),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate legacy rollback boundaries");
+
+ assert!(
+ !read_rollout(&response_path)
+ .iter()
+ .any(|line| matches!(line.item, RolloutItem::ResponseItem(_)))
+ );
+ assert!(
+ !read_rollout(&communication_path)
+ .iter()
+ .any(|line| matches!(line.item, RolloutItem::InterAgentCommunication(_)))
+ );
+ assert_eq!(
+ read_rollout(&contextual_path)
+ .into_iter()
+ .filter_map(|line| match line.item {
+ RolloutItem::ResponseItem(response) => Some(response),
+ _ => None,
+ })
+ .collect::<Vec<_>>(),
+ vec![input_response_message("user", "keep first boundary")]
+ );
+}
+
+#[tokio::test]
+async fn migration_drops_trailing_context_when_rollback_arrives_before_next_turn() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::ResponseItem(input_response_message("user", "keep question")),
+ RolloutItem::ResponseItem(input_response_message("user", "remove question")),
+ RolloutItem::ResponseItem(input_response_message(
+ "user",
+ "<turn_aborted>remove this context too</turn_aborted>",
+ )),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate trailing rollback context");
+
+ assert_eq!(
+ read_rollout(&path)
+ .into_iter()
+ .filter_map(|line| match line.item {
+ RolloutItem::ResponseItem(response) => Some(response),
+ _ => None,
+ })
+ .collect::<Vec<_>>(),
+ vec![input_response_message("user", "keep question")]
+ );
+}
+
+#[tokio::test]
+async fn migration_coalesces_response_first_user_message_rollback_boundary() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::ResponseItem(input_response_message("user", "remove question")),
+ user_message("remove question"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate response-first rollback");
+
+ assert!(
+ !read_rollout(&path)
+ .iter()
+ .any(|line| matches!(line.item, RolloutItem::ResponseItem(_)))
+ );
+}
+
+#[tokio::test]
+async fn migration_does_not_coalesce_distinct_adjacent_user_records() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::ResponseItem(input_response_message("user", "copied parent question")),
+ user_message("child question"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate distinct adjacent user records");
+
+ assert_eq!(
+ read_rollout(&path)
+ .into_iter()
+ .filter_map(|line| match line.item {
+ RolloutItem::ResponseItem(response) => Some(response),
+ _ => None,
+ })
+ .collect::<Vec<_>>(),
+ vec![input_response_message("user", "copied parent question")]
+ );
+}
+
+#[tokio::test]
+async fn migration_keeps_late_completions_for_surviving_turns_across_rollback() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("old"),
+ user_message("old question"),
+ started("remove"),
+ user_message("remove question"),
+ completed("old"),
+ exec_completion("old", "call-old"),
+ item_completed("old", "reason-old"),
+ exec_completion("remove", "call-remove"),
+ completed("remove"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ started("replacement"),
+ user_message("replacement question"),
+ completed("replacement"),
+ ],
+ );
+ let legacy_turns = build_turns_from_rollout_items(
+ &read_rollout(&path)
+ .into_iter()
+ .map(|line| line.item)
+ .collect::<Vec<_>>(),
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate late completion rollback");
+
+ let lines = read_rollout(&path);
+ assert!(!lines.iter().any(|line| matches!(
+ &line.item,
+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event))
+ if event.item.id() == "call-remove"
+ )));
+ assert!(lines.iter().any(|line| matches!(
+ &line.item,
+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event))
+ if event.turn_id == "old" && event.item.id() == "call-old"
+ )));
+ assert!(lines.iter().any(|line| matches!(
+ &line.item,
+ RolloutItem::EventMsg(EventMsg::ItemCompleted(event))
+ if event.turn_id == "old" && event.item.id() == "reason-old"
+ )));
+ assert!(lines.iter().any(|line| matches!(
+ &line.item,
+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) if event.turn_id == "old"
+ )));
+ assert!(!lines.iter().any(|line| matches!(
+ &line.item,
+ RolloutItem::EventMsg(EventMsg::TurnComplete(event)) if event.turn_id == "remove"
+ )));
+ let turns = store
+ .list_turns(ListTurnsParams {
+ thread_id,
+ include_archived: false,
+ cursor: None,
+ page_size: 10,
+ sort_direction: SortDirection::Asc,
+ items_view: StoredTurnItemsView::Summary,
+ })
+ .await
+ .expect("read late-completion rollback turns");
+ let turn_ids = turns
+ .turns
+ .iter()
+ .map(|turn| turn.turn_id.as_str())
+ .collect::<Vec<_>>();
+ assert_eq!(
+ turn_ids,
+ legacy_turns
+ .iter()
+ .map(|turn| turn.id.as_str())
+ .collect::<Vec<_>>()
+ );
+ let items = store
+ .list_items(ListItemsParams {
+ thread_id,
+ turn_id: None,
+ include_archived: false,
+ cursor: None,
+ page_size: 20,
+ sort_direction: SortDirection::Asc,
+ sort_key: ItemSortKey::CreatedAtOrdinal,
+ after_updated_at_ordinal: None,
+ })
+ .await
+ .expect("read late-completion rollback items");
+ for legacy_turn in &legacy_turns {
+ for legacy_item in &legacy_turn.items {
+ if legacy_item.id().starts_with("item-") {
+ continue;
+ }
+ assert_eq!(
+ items
+ .items
+ .iter()
+ .find(|item| item.item_id == legacy_item.id())
+ .map(|item| item.turn_id.as_str()),
+ Some(legacy_turn.id.as_str())
+ );
+ }
+ }
+}
+
+#[tokio::test]
+async fn migration_rolls_back_inter_agent_metadata_with_its_delivery() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let delivery = InterAgentCommunication::new(
+ AgentPath::root(),
+ AgentPath::root().join("worker").expect("worker path"),
+ Vec::new(),
+ "remove delivery".to_string(),
+ /*trigger_turn*/ true,
+ );
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::InterAgentCommunicationMetadata { trigger_turn: true },
+ RolloutItem::ResponseItem(delivery.to_model_input_item()),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate rolled-back inter-agent delivery");
+
+ assert!(!read_rollout(&path).iter().any(|line| {
+ matches!(
+ line.item,
+ RolloutItem::InterAgentCommunicationMetadata { .. }
+ | RolloutItem::ResponseItem(ResponseItem::AgentMessage { .. })
+ )
+ }));
+}
+
+#[tokio::test]
+async fn migration_rolls_back_pre_compaction_turns_from_sqlite_history() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("keep-before-compaction"),
+ user_message("old question"),
+ completed("keep-before-compaction"),
+ compacted(vec![
+ input_response_message("user", "old question"),
+ ResponseItem::Message {
+ id: None,
+ role: "assistant".to_string(),
+ content: vec![ContentItem::OutputText {
+ text: "old answer".to_string(),
+ }],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ },
+ ]),
+ started("remove-after-compaction"),
+ user_message("new question"),
+ completed("remove-after-compaction"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 2,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate rollback through compaction");
+
+ let turns = store
+ .list_turns(ListTurnsParams {
+ thread_id,
+ include_archived: false,
+ cursor: None,
+ page_size: 10,
+ sort_direction: SortDirection::Asc,
+ items_view: StoredTurnItemsView::Summary,
+ })
+ .await
+ .expect("read rollback-through-compaction turns");
+ assert_eq!(turns.turns.len(), 1);
+ let replacement_history = read_rollout(&path)
+ .into_iter()
+ .find_map(|line| match line.item {
+ RolloutItem::Compacted(item) => item.replacement_history,
+ _ => None,
+ })
+ .expect("retained compaction");
+ assert!(replacement_history.is_empty());
+}
+
+#[tokio::test]
+async fn migration_preserves_reverse_replay_anchor_after_pre_compaction_rollback() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ RolloutItem::ResponseItem(input_response_message("user", "remove question")),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ compacted(vec![input_response_message("user", "old question")]),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate pre-compaction rollback");
+
+ let replacement_history = read_rollout(&path)
+ .into_iter()
+ .find_map(|line| match line.item {
+ RolloutItem::Compacted(item) => item.replacement_history,
+ _ => None,
+ })
+ .expect("retained compaction");
+ assert!(replacement_history.is_empty());
+}
+
+#[tokio::test]
+async fn migration_keeps_empty_replay_anchor_from_rolled_back_turn() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("keep"),
+ user_message("keep question"),
+ completed("keep"),
+ started("remove"),
+ user_message("remove question"),
+ compacted(vec![input_response_message("user", "old question")]),
+ completed("remove"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ started("replacement"),
+ user_message("replacement question"),
+ completed("replacement"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate rolled-back replay anchor");
+
+ let replacement_history = read_rollout(&path)
+ .into_iter()
+ .find_map(|line| match line.item {
+ RolloutItem::Compacted(item) => item.replacement_history,
+ _ => None,
+ })
+ .expect("retained compaction");
+ assert!(replacement_history.is_empty());
+}
+
+#[tokio::test]
+async fn migration_uses_turn_context_to_select_reverse_replay_anchor() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("keep"),
+ user_message("keep question"),
+ compacted(vec![input_response_message("user", "keep question")]),
+ completed("keep"),
+ started("remove"),
+ user_message("remove question"),
+ serde_json::from_value(json!({
+ "type": "event_msg",
+ "payload": {
+ "type": "turn_aborted",
+ "turn_id": "other",
+ "reason": "interrupted"
+ }
+ }))
+ .expect("build mismatched turn abort"),
+ serde_json::from_value(json!({
+ "type": "turn_context",
+ "payload": {
+ "turn_id": "remove",
+ "cwd": home.path(),
+ "approval_policy": "never",
+ "sandbox_policy": {"type": "read-only"},
+ "model": "test-model",
+ "summary": "auto"
+ }
+ }))
+ .expect("build turn context"),
+ compacted(vec![input_response_message("user", "remove question")]),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ started("replacement"),
+ user_message("replacement question"),
+ completed("replacement"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate turn-context replay anchor");
+
+ let replacement_histories = read_rollout(&path)
+ .into_iter()
+ .filter_map(|line| match line.item {
+ RolloutItem::Compacted(item) => item.replacement_history,
+ _ => None,
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(
+ replacement_histories,
+ vec![vec![input_response_message("user", "keep question")]]
+ );
+}
+
+#[tokio::test]
+async fn migration_applies_cumulative_and_overflowing_rollbacks() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ started("first"),
+ user_message("first question"),
+ completed("first"),
+ started("second"),
+ user_message("second question"),
+ completed("second"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 1,
+ })),
+ started("third"),
+ user_message("third question"),
+ completed("third"),
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
+ num_turns: 99,
+ })),
+ user_message("replacement question"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate cumulative rollbacks");
+
+ let turns = store
+ .list_turns(ListTurnsParams {
+ thread_id,
+ include_archived: false,
+ cursor: None,
+ page_size: 10,
+ sort_direction: SortDirection::Asc,
+ items_view: StoredTurnItemsView::Summary,
+ })
+ .await
+ .expect("read cumulative rollback turns");
+ assert_eq!(turns.turns.len(), 1);
+ assert!(!read_rollout(&path).iter().any(|line| matches!(
+ line.item,
+ RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))
+ )));
+}
+
+#[tokio::test]
+async fn migration_drops_copied_user_fork_metadata_without_creating_a_history_base() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let parent_id = ThreadId::new();
+ let copied_metadata = SessionMeta {
+ session_id: parent_id.into(),
+ id: parent_id,
+ timestamp: TIMESTAMP.to_string(),
+ cwd: home.path().to_path_buf(),
+ source: SessionSource::Cli,
+ ..SessionMeta::default()
+ };
+ let copied_response =
+ RolloutItem::ResponseItem(input_response_message("user", "copied parent history"));
+ let path = write_rollout_with_fork(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ Some(parent_id),
+ vec![
+ RolloutItem::SessionMeta(SessionMetaLine {
+ meta: copied_metadata,
+ git: None,
+ }),
+ copied_response,
+ user_message("child question"),
+ agent_message("child answer"),
+ ],
+ );
+ let expected_responses = read_rollout(&path)
+ .into_iter()
+ .filter_map(|line| match line.item {
+ RolloutItem::ResponseItem(item) => {
+ Some(serde_json::to_value(item).expect("serialize copied response"))
+ }
+ _ => None,
+ })
+ .collect::<Vec<_>>();
+ let store = indexed_store(home.path()).await;
+
+ let report = store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate copied user fork");
+
+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
+ let lines = read_rollout(&path);
+ assert!(matches!(
+ &lines[0].item,
+ RolloutItem::SessionMeta(metadata)
+ if metadata.meta.id == thread_id
+ && metadata.meta.forked_from_id == Some(parent_id)
+ && metadata.meta.history_mode == ThreadHistoryMode::Paginated
+ && metadata.meta.history_base.is_none()
+ ));
+ assert_eq!(
+ lines
+ .iter()
+ .filter(|line| matches!(line.item, RolloutItem::SessionMeta(_)))
+ .count(),
+ 1
+ );
+ assert_eq!(
+ lines
+ .into_iter()
+ .filter_map(|line| match line.item {
+ RolloutItem::ResponseItem(item) => {
+ Some(serde_json::to_value(item).expect("serialize migrated response"))
+ }
+ _ => None,
+ })
+ .collect::<Vec<_>>(),
+ expected_responses
+ );
+}
+
+#[tokio::test]
+async fn migration_compacts_subagent_prefix_and_does_not_project_it() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::SubAgent(SubAgentSource::Other("test".to_string())),
+ vec![
+ RolloutItem::Compacted(CompactedItem {
+ message: "superseded checkpoint".repeat(1024),
+ replacement_history: Some(Vec::new()),
+ window_number: Some(1),
+ first_window_id: None,
+ previous_window_id: None,
+ window_id: None,
+ }),
+ RolloutItem::Compacted(CompactedItem {
+ message: "latest checkpoint".to_string(),
+ replacement_history: Some(vec![ResponseItem::Message {
+ id: None,
+ role: "user".to_string(),
+ content: vec![ContentItem::InputText {
+ text: "latest compacted context".to_string(),
+ }],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ }]),
+ window_number: Some(2),
+ first_window_id: None,
+ previous_window_id: None,
+ window_id: None,
+ }),
+ started("child-turn"),
+ RolloutItem::TurnContext(TurnContextItem {
+ turn_id: Some("child-turn".to_string()),
+ cwd: serde_json::from_value(json!(home.path())).expect("absolute cwd"),
+ workspace_roots: None,
+ current_date: None,
+ timezone: None,
+ approval_policy: AskForApproval::Never,
+ approvals_reviewer: None,
+ sandbox_policy: SandboxPolicy::new_read_only_policy(),
+ permission_profile: None,
+ network: None,
+ file_system_sandbox_policy: None,
+ model: "test-model".to_string(),
+ comp_hash: None,
+ personality: None,
+ collaboration_mode: None,
+ multi_agent_version: None,
+ multi_agent_mode: None,
+ realtime_active: None,
+ effort: None,
+ summary: ReasoningSummary::Auto,
+ }),
+ user_message("child question"),
+ agent_message("child answer"),
+ completed("child-turn"),
+ ],
+ );
+ writeln!(
+ fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .expect("open legacy subagent rollout"),
+ "{{not valid rollout json"
+ )
+ .expect("append malformed record");
+ let store = indexed_store(home.path()).await;
+
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate legacy subagent");
+
+ let lines = read_rollout(&path);
+ let RolloutItem::SessionMeta(metadata) = &lines[0].item else {
+ panic!("migrated rollout should start with session metadata");
+ };
+ assert_eq!(
+ metadata.meta.subagent_history_start_ordinal,
+ Some(lines.len() as u64)
+ );
+ assert!(
+ !fs::read_to_string(&path)
+ .expect("read migrated rollout")
+ .contains("superseded checkpoint")
+ );
+ let context = store
+ .load_latest_model_context(LoadThreadHistoryParams {
+ thread_id,
+ include_archived: false,
+ })
+ .await
+ .expect("load migrated model context");
+ assert!(context.items.iter().any(|item| {
+ matches!(item, RolloutItem::Compacted(compacted) if compacted.message == "latest checkpoint")
+ }));
+ assert!(
+ list_active_summary_turns(&store, thread_id)
+ .await
+ .turns
+ .is_empty()
+ );
+}
+
+#[tokio::test]
+async fn migration_keeps_small_uncompacted_subagent_replay_as_prefix() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
let path = write_rollout(
home.path(),
thread_id,
- SessionSource::Cli,
- vec![user_message("question")],
+ SessionSource::SubAgent(SubAgentSource::Other("test".to_string())),
+ vec![
+ user_message("child question"),
+ agent_message("child answer"),
+ ],
);
- let final_line = RolloutLine {
- timestamp: TIMESTAMP.to_string(),
- ordinal: None,
- item: agent_message("answer"),
- };
- fs::OpenOptions::new()
- .append(true)
- .open(&path)
- .expect("open legacy rollout")
- .write_all(
- serde_json::to_string(&final_line)
- .expect("serialize final record")
- .as_bytes(),
- )
- .expect("append final record");
let store = indexed_store(home.path()).await;
store
.migrate_rollouts(apply_options())
.await
- .expect("migrate legacy rollout");
+ .expect("migrate uncompacted legacy subagent");
+ let lines = read_rollout(&path);
+ let RolloutItem::SessionMeta(metadata) = &lines[0].item else {
+ panic!("migrated rollout should start with session metadata");
+ };
assert_eq!(
- read_rollout(&path)
+ metadata.meta.subagent_history_start_ordinal,
+ Some(lines.len() as u64)
+ );
+ assert_eq!(
+ lines
.iter()
.filter(|line| matches!(line.item, RolloutItem::EventMsg(EventMsg::ItemCompleted(_))))
.count(),
@@ -257,90 +1426,39 @@ async fn migration_preserves_valid_final_record_without_newline() {
}
#[tokio::test]
-async fn migration_preserves_copied_user_fork_history_without_creating_a_history_base() {
+async fn migration_projects_memory_consolidation_as_ordinary_history() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
- let parent_id = ThreadId::new();
- let copied_metadata = SessionMeta {
- session_id: parent_id.into(),
- id: parent_id,
- timestamp: TIMESTAMP.to_string(),
- cwd: home.path().to_path_buf(),
- source: SessionSource::Cli,
- ..SessionMeta::default()
- };
- let copied_response = RolloutItem::ResponseItem(ResponseItem::Message {
- id: None,
- role: "user".to_string(),
- content: vec![ContentItem::InputText {
- text: "copied parent history".to_string(),
- }],
- phase: None,
- internal_chat_message_metadata_passthrough: None,
- });
- let path = write_rollout_with_fork(
+ let path = write_rollout(
home.path(),
thread_id,
- SessionSource::Cli,
- Some(parent_id),
+ SessionSource::SubAgent(SubAgentSource::MemoryConsolidation),
vec![
- RolloutItem::SessionMeta(SessionMetaLine {
- meta: copied_metadata,
- git: None,
- }),
- copied_response,
- user_message("child question"),
- agent_message("child answer"),
+ user_message("memory question"),
+ agent_message("memory answer"),
],
);
- let expected_responses = read_rollout(&path)
- .into_iter()
- .filter_map(|line| match line.item {
- RolloutItem::ResponseItem(item) => {
- Some(serde_json::to_value(item).expect("serialize copied response"))
- }
- _ => None,
- })
- .collect::<Vec<_>>();
let store = indexed_store(home.path()).await;
- let report = store
+ store
.migrate_rollouts(apply_options())
.await
- .expect("migrate copied user fork");
+ .expect("migrate memory consolidation rollout");
- assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
let lines = read_rollout(&path);
assert!(matches!(
&lines[0].item,
RolloutItem::SessionMeta(metadata)
- if metadata.meta.id == thread_id
- && metadata.meta.forked_from_id == Some(parent_id)
- && metadata.meta.history_mode == ThreadHistoryMode::Paginated
- && metadata.meta.history_base.is_none()
- ));
- assert!(matches!(
- &lines[1].item,
- RolloutItem::SessionMeta(metadata)
- if metadata.meta.id == parent_id
- && metadata.meta.history_mode == ThreadHistoryMode::Legacy
+ if metadata.meta.history_mode == ThreadHistoryMode::Paginated
+ && metadata.meta.subagent_history_start_ordinal.is_none()
));
- assert_eq!(
- lines
- .into_iter()
- .filter_map(|line| match line.item {
- RolloutItem::ResponseItem(item) => {
- Some(serde_json::to_value(item).expect("serialize migrated response"))
- }
- _ => None,
- })
- .collect::<Vec<_>>(),
- expected_responses
- );
+ let turns = list_active_summary_turns(&store, thread_id).await;
+ assert_eq!(turns.turns.len(), 1);
+ assert_eq!(turns.turns[0].items.len(), 2);
}
#[tokio::test]
-async fn dry_run_reports_newest_first_and_skips_subagents() {
+async fn dry_run_reports_migration_order() {
let home = TempDir::new().expect("create Codex home");
let root_id = ThreadId::new();
let root = write_rollout(
@@ -349,6 +1467,11 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {
SessionSource::Cli,
vec![user_message("root question")],
);
+ let newest_directory = home.path().join("sessions/2025/01/04");
+ fs::create_dir_all(&newest_directory).expect("create newest rollout directory");
+ let newest_root = newest_directory.join(format!("rollout-2025-01-04T12-00-00-{root_id}.jsonl"));
+ fs::rename(&root, &newest_root).expect("move root rollout to newest date");
+ let root = newest_root;
let subagent_id = ThreadId::new();
let subagent = write_rollout(
home.path(),
@@ -356,13 +1479,40 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {
SessionSource::SubAgent(SubAgentSource::Other("test".to_string())),
vec![user_message("subagent question")],
);
+ let memory_id = ThreadId::new();
+ let memory = write_rollout(
+ home.path(),
+ memory_id,
+ SessionSource::SubAgent(SubAgentSource::MemoryConsolidation),
+ vec![user_message("memory question")],
+ );
+ let memory_directory = home.path().join("sessions/2025/01/01");
+ fs::create_dir_all(&memory_directory).expect("create memory rollout directory");
+ let moved_memory = memory_directory.join(memory.file_name().expect("memory filename"));
+ fs::rename(memory, &moved_memory).expect("move memory rollout");
let compressed_id = ThreadId::new();
- let compressed = compress_rollout(write_rollout(
+ let oldest_directory = home.path().join("sessions/2025/01/02");
+ fs::create_dir_all(&oldest_directory).expect("create oldest rollout directory");
+ let source_compressed_plain = write_rollout(
home.path(),
compressed_id,
SessionSource::Cli,
vec![user_message("compressed question")],
- ));
+ );
+ let compressed_plain =
+ oldest_directory.join(format!("rollout-2025-01-02T12-00-00-{compressed_id}.jsonl"));
+ fs::rename(source_compressed_plain, &compressed_plain).expect("move compressed rollout");
+ let compressed = compress_rollout(&compressed_plain);
+ let archived_id = ThreadId::new();
+ let archived = move_to_archived(
+ home.path(),
+ write_rollout(
+ home.path(),
+ archived_id,
+ SessionSource::Cli,
+ vec![user_message("archived question")],
+ ),
+ );
let original = fs::read(&root).expect("read original root rollout");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
@@ -371,16 +1521,13 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {
.await
.expect("inspect legacy rollouts");
- let mut expected = vec![
+ let expected = vec![
(root.clone(), root_id, RolloutMigrationStatus::Eligible),
- (
- subagent,
- subagent_id,
- RolloutMigrationStatus::SkippedSubagent,
- ),
+ (subagent, subagent_id, RolloutMigrationStatus::Eligible),
(compressed, compressed_id, RolloutMigrationStatus::Eligible),
+ (moved_memory, memory_id, RolloutMigrationStatus::Eligible),
+ (archived, archived_id, RolloutMigrationStatus::Eligible),
];
- expected.sort_by(|(left, ..), (right, ..)| right.cmp(left));
assert_eq!(
report
.outcomes
@@ -394,27 +1541,46 @@ async fn dry_run_reports_newest_first_and_skips_subagents() {
expected,
);
assert_eq!(fs::read(&root).expect("read inspected rollout"), original);
+
+ let selected = store
+ .migrate_rollouts(RolloutMigrationOptions {
+ thread_ids: vec![root_id],
+ ..RolloutMigrationOptions::default()
+ })
+ .await
+ .expect("inspect selected rollout");
+ assert_eq!(selected.outcomes.len(), 1);
+ assert_eq!(selected.outcomes[0].thread_id, Some(root_id));
+ assert_eq!(
+ selected.outcomes[0].status,
+ RolloutMigrationStatus::Eligible
+ );
}
#[tokio::test]
async fn migration_preserves_compressed_rollouts_during_publish_and_recovery() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
- let compressed_path = compress_rollout(write_rollout(
+ let path = write_rollout(
home.path(),
thread_id,
SessionSource::Cli,
- vec![user_message("compressed question")],
- ));
- let plain_path = compressed_path.with_extension("");
+ vec![
+ user_message("compressed question"),
+ agent_message("compressed answer"),
+ ],
+ );
+ let compressed_path = compress_rollout(&path);
let store = indexed_store(home.path()).await;
let report = store
.migrate_rollouts(apply_options())
.await
.expect("migrate compressed rollout");
+
assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
- assert!(!plain_path.exists());
+ assert!(!path.exists());
+ assert!(compressed_path.exists());
assert_eq!(
codex_rollout::read_session_meta_line(&compressed_path)
.await
@@ -423,6 +1589,9 @@ async fn migration_preserves_compressed_rollouts_during_publish_and_recovery() {
.history_mode,
ThreadHistoryMode::Paginated
);
+ let turns = list_active_summary_turns(&store, thread_id).await;
+ assert_eq!(turns.turns.len(), 1);
+ assert_eq!(turns.turns[0].items.len(), 2);
thread_history::delete_thread(&store, thread_id)
.await
@@ -442,16 +1611,60 @@ async fn migration_preserves_compressed_rollouts_during_publish_and_recovery() {
);
assert!(recovered.outcomes[0].bytes_processed > 0);
assert!(compressed_path.exists());
- assert!(!plain_path.exists());
+ assert!(!path.exists());
assert!(!journal_path.exists());
}
+#[tokio::test]
+async fn migration_migrates_archived_rollouts_without_unarchiving_them() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let active_path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ user_message("archived question"),
+ agent_message("archived answer"),
+ ],
+ );
+ let archived_path = move_to_archived(home.path(), active_path.clone());
+ let store = indexed_store(home.path()).await;
+
+ let report = store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("migrate archived rollout");
+
+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
+ assert!(!active_path.exists());
+ assert!(archived_path.exists());
+ assert!(matches!(
+ &read_rollout(&archived_path)[0].item,
+ RolloutItem::SessionMeta(metadata)
+ if metadata.meta.history_mode == ThreadHistoryMode::Paginated
+ ));
+ let turns = store
+ .list_turns(ListTurnsParams {
+ thread_id,
+ include_archived: true,
+ cursor: None,
+ page_size: 10,
+ sort_direction: SortDirection::Asc,
+ items_view: StoredTurnItemsView::Summary,
+ })
+ .await
+ .expect("read archived projected turns");
+ assert_eq!(turns.turns.len(), 1);
+ assert_eq!(turns.turns[0].items.len(), 2);
+}
+
#[cfg(unix)]
#[tokio::test]
async fn decompression_temporaries_are_owner_only() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
- let compressed_path = compress_rollout(write_rollout(
+ let compressed_path = compress_rollout(&write_rollout(
home.path(),
thread_id,
SessionSource::Cli,
@@ -502,6 +1715,31 @@ async fn migration_skips_threads_with_an_active_writer() {
assert_eq!(fs::read(&path).expect("read unmodified rollout"), original);
}
+#[tokio::test]
+async fn migration_apply_conflicts_with_rollout_maintenance() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![user_message("maintenance question")],
+ );
+ let original = fs::read(&path).expect("read legacy rollout");
+ let _maintenance_guard = codex_rollout::try_acquire_rollout_maintenance_lock(home.path())
+ .expect("acquire rollout maintenance lock")
+ .expect("claim rollout maintenance lock");
+ let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
+
+ let error = store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect_err("reject concurrent rollout maintenance");
+
+ assert!(matches!(error, ThreadStoreError::Conflict { .. }));
+ assert_eq!(fs::read(&path).expect("read untouched rollout"), original);
+}
+
#[tokio::test]
async fn migration_recovers_a_published_rollout_with_missing_projection() {
let home = TempDir::new().expect("create Codex home");
@@ -619,49 +1857,193 @@ async fn migration_recovers_pending_rollouts_before_new_work() {
}
#[tokio::test]
-async fn failed_migration_preserves_the_legacy_rollout_and_can_be_retried() {
+async fn migration_recovers_a_compressed_published_rollout() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![
+ user_message("recover compressed question"),
+ agent_message("recover compressed answer"),
+ ],
+ );
+ let store = indexed_store(home.path()).await;
+ store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("publish canonical rollout");
+ thread_history::delete_thread(&store, thread_id)
+ .await
+ .expect("simulate interrupted projection");
+ let state_db = store.state_db.as_ref().expect("state db");
+ let mut legacy_metadata = state_db
+ .get_thread(thread_id)
+ .await
+ .expect("read thread metadata")
+ .expect("thread metadata");
+ legacy_metadata.history_mode = ThreadHistoryMode::Legacy;
+ state_db
+ .delete_thread(thread_id)
+ .await
+ .expect("remove paginated thread metadata");
+ assert!(
+ state_db
+ .insert_thread_if_absent(&legacy_metadata)
+ .await
+ .expect("restore legacy thread metadata")
+ );
+ let journal_path = migration_journal_path(home.path(), thread_id);
+ write_migration_journal(&journal_path)
+ .await
+ .expect("simulate pending migration journal");
+ let compressed_path = compress_rollout(&path);
+
+ let report = store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("recover compressed published rollout");
+
+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
+ assert!(!path.exists());
+ assert!(compressed_path.exists());
+ assert!(!journal_path.exists());
+ assert_eq!(
+ store
+ .state_db
+ .as_ref()
+ .expect("state db")
+ .get_thread(thread_id)
+ .await
+ .expect("read repaired thread metadata")
+ .expect("thread metadata")
+ .history_mode,
+ ThreadHistoryMode::Paginated
+ );
+ let projection = thread_history::projection_state(&store, thread_id)
+ .await
+ .expect("read repaired projection")
+ .expect("projection was rebuilt");
+ assert_eq!(
+ projection.next_byte_offset,
+ zstd::stream::decode_all(fs::File::open(&compressed_path).expect("open rollout"))
+ .expect("decompress rollout")
+ .len() as u64
+ );
+}
+
+#[tokio::test]
+async fn migration_skips_oversized_jsonl_records() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
let path = write_rollout(
home.path(),
thread_id,
SessionSource::Cli,
- vec![user_message("recoverable question")],
+ vec![user_message("kept question")],
);
- let valid_length = fs::metadata(&path)
- .expect("read valid rollout length")
- .len();
+ let oversized_output = "x".repeat(super::MAX_ROLLOUT_LINE_BYTES);
let mut file = fs::OpenOptions::new()
.append(true)
.open(&path)
.expect("open legacy rollout");
- writeln!(file, "{{not valid rollout json").expect("append malformed record");
+ writeln!(
+ file,
+ "{{\"timestamp\":\"{TIMESTAMP}\",\"type\":\"response_item\",\"payload\":{{\"type\":\"function_call_output\",\"call_id\":\"call-1\",\"output\":\"{oversized_output}\"}}}}"
+ )
+ .expect("write oversized rollout record");
drop(file);
- let original = fs::read(&path).expect("read malformed legacy rollout");
let store = indexed_store(home.path()).await;
let report = store
.migrate_rollouts(apply_options())
.await
- .expect("report malformed legacy rollout");
+ .expect("migrate oversized rollout record");
+
+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
+ assert!(
+ fs::metadata(&path)
+ .expect("read migrated rollout metadata")
+ .len()
+ < super::MAX_ROLLOUT_LINE_BYTES as u64
+ );
+ let turns = list_active_summary_turns(&store, thread_id).await;
+ assert_eq!(turns.turns.len(), 1);
+ assert_eq!(turns.turns[0].items.len(), 1);
+}
+
+#[tokio::test]
+async fn migration_skips_empty_rollout_files() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let directory = home.path().join("sessions/2025/01/03");
+ fs::create_dir_all(&directory).expect("create rollout directory");
+ let path = directory.join(format!("rollout-2025-01-03T12-00-00-{thread_id}.jsonl"));
+ fs::File::create(&path).expect("create empty rollout");
+ let store = indexed_store(home.path()).await;
+
+ let report = store
+ .migrate_rollouts(apply_options())
+ .await
+ .expect("inspect empty rollout");
- assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Failed);
assert_eq!(
- fs::read(&path).expect("read preserved legacy rollout"),
- original
+ report.outcomes[0].status,
+ RolloutMigrationStatus::SkippedEmpty
+ );
+ assert_eq!(fs::metadata(&path).expect("read empty rollout").len(), 0);
+ assert!(
+ store
+ .state_db
+ .as_ref()
+ .expect("state db")
+ .get_thread(thread_id)
+ .await
+ .expect("read thread metadata")
+ .is_none()
);
+}
- fs::OpenOptions::new()
- .write(true)
+#[tokio::test]
+async fn migration_skips_malformed_lines_and_trailing_partial_tail() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![user_message("kept question")],
+ );
+ let mut file = fs::OpenOptions::new()
+ .append(true)
.open(&path)
- .expect("open repaired legacy rollout")
- .set_len(valid_length)
- .expect("remove malformed tail");
- let retry = store
+ .expect("open legacy rollout");
+ writeln!(file, "{{not valid rollout json").expect("append malformed record");
+ let later_valid_line = RolloutLine {
+ timestamp: TIMESTAMP.to_string(),
+ ordinal: None,
+ item: agent_message("kept answer"),
+ };
+ writeln!(
+ file,
+ "{}",
+ serde_json::to_string(&later_valid_line).expect("serialize later valid record")
+ )
+ .expect("append later valid record");
+ file.write_all(br#"{"timestamp":"unterminated""#)
+ .expect("append partial tail");
+ drop(file);
+ let store = indexed_store(home.path()).await;
+
+ let report = store
.migrate_rollouts(apply_options())
.await
- .expect("retry repaired legacy rollout");
+ .expect("migrate malformed legacy rollout");
- assert_eq!(retry.outcomes[0].status, RolloutMigrationStatus::Migrated);
+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
assert!(!migration_journal_path(home.path(), thread_id).exists());
+ let turns = list_active_summary_turns(&store, thread_id).await;
+ assert_eq!(turns.turns.len(), 1);
+ assert_eq!(turns.turns[0].items.len(), 2);
}