Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Harden startup rollout migration against concurrent updates (#40499)

Owen Lin465eafacbc2d
diff --git a/codex-rs/thread-store/src/local/rollout_migration.rs b/codex-rs/thread-store/src/local/rollout_migration.rs
index 028b6d0094..46d71d5f11 100644
--- a/codex-rs/thread-store/src/local/rollout_migration.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration.rs
@@ -81,6 +81,11 @@ enum CanonicalizationAttempt {
NeedsRollbackPlan,
}
+enum RolloutMigrationPaths {
+ Discover,
+ Known(Vec<PathBuf>),
+}
+
struct CanonicalizationSource<'a> {
thread_id: ThreadId,
source_path: &'a Path,
@@ -268,6 +273,7 @@ impl LocalThreadStore {
options,
|_| {},
RolloutMigrationTrigger::Manual,
+ RolloutMigrationPaths::Discover,
)
.await
}
@@ -282,6 +288,7 @@ impl LocalThreadStore {
options,
on_progress,
RolloutMigrationTrigger::Manual,
+ RolloutMigrationPaths::Discover,
)
.await
}
@@ -291,10 +298,11 @@ impl LocalThreadStore {
options: RolloutMigrationOptions,
mut on_progress: impl FnMut(RolloutMigrationProgress),
trigger: RolloutMigrationTrigger,
+ paths: RolloutMigrationPaths,
) -> ThreadStoreResult<RolloutMigrationReport> {
let telemetry = RolloutMigrationTelemetry::new(trigger, &options);
let result = self
- .migrate_rollouts_with_progress_inner(options, &mut on_progress)
+ .migrate_rollouts_with_progress_inner(options, &mut on_progress, paths)
.await;
telemetry.finish(&result);
result
@@ -304,6 +312,7 @@ impl LocalThreadStore {
&self,
options: RolloutMigrationOptions,
on_progress: &mut impl FnMut(RolloutMigrationProgress),
+ paths: RolloutMigrationPaths,
) -> ThreadStoreResult<RolloutMigrationReport> {
let mut limiter = RolloutMigrationRateLimiter::new(options.max_mib_per_second)?;
let _maintenance_guard = match options.mode {
@@ -317,18 +326,12 @@ impl LocalThreadStore {
})?,
),
};
- let mut paths =
- find_rollout_paths(&self.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR))
- .await?;
- paths.extend(
- find_rollout_paths(
- &self
- .config
- .codex_home
- .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR),
- )
- .await?,
- );
+ let mut paths = match paths {
+ RolloutMigrationPaths::Discover => {
+ find_all_rollout_paths(&self.config.codex_home).await?
+ }
+ RolloutMigrationPaths::Known(paths) => paths,
+ };
if options.mode == RolloutMigrationMode::Apply {
let pending_thread_ids = pending_migration_thread_ids(&self.config.codex_home).await?;
paths.sort_by_key(|path| {
@@ -376,36 +379,76 @@ impl LocalThreadStore {
legacy_names: &HashMap<ThreadId, String>,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<Option<RolloutMigrationOutcome>> {
- let metadata = match codex_rollout::read_session_meta_line(&path).await {
- Ok(metadata) => metadata,
- Err(error) => {
- let thread_id = thread_id_from_rollout_filename(&path);
- if !matches_selection(&options.thread_ids, thread_id) {
- return Ok(None);
+ let mut retried_moved_path = false;
+ let metadata = loop {
+ let error = match codex_rollout::read_session_meta_line(&path).await {
+ Ok(metadata) => break metadata,
+ Err(error) if !retried_moved_path && error.kind() == io::ErrorKind::NotFound => {
+ // A different Codex process can archive or compress this rollout after path
+ // discovery but before we take its writer lock. Retry the same rollout under
+ // its current root/suffix before treating the missing snapshot path as failed.
+ if let Some(current_path) =
+ find_current_rollout_path(&self.config.codex_home, &path).await?
+ {
+ path = current_path;
+ retried_moved_path = true;
+ continue;
+ }
+ error
}
- let empty = tokio::fs::metadata(&path)
- .await
- .is_ok_and(|metadata| metadata.len() == 0);
- let failure_reason = if empty {
- None
- } else if error.kind() != io::ErrorKind::Other {
- Some(RolloutMigrationFailureReason::RolloutReadFailed)
- } else {
- Some(RolloutMigrationFailureReason::InvalidSessionMetadata)
- };
- return Ok(Some(RolloutMigrationOutcome {
- thread_id,
- rollout_path: path,
- status: if empty {
- RolloutMigrationStatus::SkippedEmpty
- } else {
- RolloutMigrationStatus::Failed
- },
- failure_reason,
- bytes_processed: 0,
- message: (!empty).then(|| error.to_string()),
- }));
+ Err(error) => error,
+ };
+ let thread_id = thread_id_from_rollout_filename(&path);
+ if !matches_selection(&options.thread_ids, thread_id) {
+ return Ok(None);
}
+ let initially_empty = tokio::fs::metadata(&path)
+ .await
+ .is_ok_and(|metadata| metadata.len() == 0);
+ // Writers create the rollout path before SessionMeta is durable. Re-read under the
+ // writer lock so a writer that just finished does not become a permanent empty skip.
+ let error = if initially_empty
+ && options.mode == RolloutMigrationMode::Apply
+ && let Some(thread_id) = thread_id
+ {
+ let _writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {
+ Ok(guard) => guard,
+ Err(ThreadStoreError::Conflict { message }) => {
+ return Ok(Some(skipped_busy_outcome(
+ thread_id, path, message, /*bytes_processed*/ 0,
+ )));
+ }
+ Err(error) => return Err(error),
+ };
+ match codex_rollout::read_session_meta_line(&path).await {
+ Ok(metadata) => break metadata,
+ Err(error) => error,
+ }
+ } else {
+ error
+ };
+ let empty = tokio::fs::metadata(&path)
+ .await
+ .is_ok_and(|metadata| metadata.len() == 0);
+ let failure_reason = if empty {
+ None
+ } else if error.kind() != io::ErrorKind::Other {
+ Some(RolloutMigrationFailureReason::RolloutReadFailed)
+ } else {
+ Some(RolloutMigrationFailureReason::InvalidSessionMetadata)
+ };
+ return Ok(Some(RolloutMigrationOutcome {
+ thread_id,
+ rollout_path: path,
+ status: if empty {
+ RolloutMigrationStatus::SkippedEmpty
+ } else {
+ RolloutMigrationStatus::Failed
+ },
+ failure_reason,
+ bytes_processed: 0,
+ message: (!empty).then(|| error.to_string()),
+ }));
};
let thread_id = metadata.meta.id;
if !matches_selection(&options.thread_ids, Some(thread_id)) {
@@ -511,6 +554,16 @@ impl LocalThreadStore {
)));
}
};
+ // SessionMeta gives us the writer-lock id, so archiving can win between that read and
+ // lock acquisition. Once the lock is ours, follow the same rollout to its current path.
+ if !tokio::fs::try_exists(&path)
+ .await
+ .map_err(migration_error)?
+ && let Some(current_path) =
+ find_current_rollout_path(&self.config.codex_home, &path).await?
+ {
+ path = current_path;
+ }
let bytes_before = limiter.bytes_processed;
let result = match self
.migrate_one_rollout(thread_id, &path, &journal_path, kind, legacy_names, limiter)
@@ -1228,6 +1281,30 @@ async fn find_rollout_paths(root: &Path) -> ThreadStoreResult<Vec<PathBuf>> {
Ok(paths)
}
+async fn find_all_rollout_paths(codex_home: &Path) -> ThreadStoreResult<Vec<PathBuf>> {
+ let mut paths = find_rollout_paths(&codex_home.join(codex_rollout::SESSIONS_SUBDIR)).await?;
+ paths.extend(
+ find_rollout_paths(&codex_home.join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR)).await?,
+ );
+ Ok(paths)
+}
+
+async fn find_current_rollout_path(
+ codex_home: &Path,
+ stale_path: &Path,
+) -> ThreadStoreResult<Option<PathBuf>> {
+ let plain_path = codex_rollout::plain_rollout_path(stale_path);
+ let Some(file_name) = plain_path.file_name() else {
+ return Ok(None);
+ };
+ Ok(find_all_rollout_paths(codex_home)
+ .await?
+ .into_iter()
+ .find(|candidate| {
+ codex_rollout::plain_rollout_path(candidate).file_name() == Some(file_name)
+ }))
+}
+
fn matches_selection(selected: &[ThreadId], actual: Option<ThreadId>) -> bool {
selected.is_empty() || actual.is_some_and(|thread_id| selected.contains(&thread_id))
}
diff --git a/codex-rs/thread-store/src/local/rollout_migration/startup.rs b/codex-rs/thread-store/src/local/rollout_migration/startup.rs
index 18c467526c..dbc48ed8c0 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/startup.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/startup.rs
@@ -4,13 +4,15 @@
//! rollout files on later launches. When it finds legacy history or a pending recovery marker, it
//! invokes the existing full migration path.
//!
-//! Empty or malformed rollouts are fingerprinted so they do not block the cursor forever. If one
-//! changes later, startup retries it instead of trusting the old skip.
+//! Rollouts that background migration cannot finish are remembered so they do not hold the cursor
+//! back forever. Ordinary failures stay skipped until a manual migration retries them; busy
+//! rollouts are retried on later startups because the writer may have gone away.
use std::collections::HashSet;
-use std::io::ErrorKind;
+use std::ffi::OsString;
use std::path::Path;
use std::path::PathBuf;
+use std::time::Duration;
use std::time::SystemTime;
use chrono::NaiveDateTime;
@@ -23,19 +25,26 @@ use codex_state::RolloutMigrationSkippedRollout;
use super::LocalThreadStore;
use super::RolloutMigrationMode;
use super::RolloutMigrationOptions;
+use super::RolloutMigrationReport;
use super::RolloutMigrationStatus;
-use super::find_rollout_paths;
+use super::find_all_rollout_paths;
use super::migration_error;
+use super::publish::migration_journal_path;
use super::publish::pending_migration_thread_ids;
use super::telemetry::RolloutMigrationTrigger;
+use super::thread_id_from_rollout_filename;
+use crate::ThreadStoreError;
use crate::ThreadStoreResult;
const LEGACY_TO_PAGINATED_MIGRATION_ID: &str = "legacy_to_paginated_v1";
const EMPTY_SKIP_REASON: &str = "empty";
+const FAILED_SKIP_REASON: &str = "failed";
const MALFORMED_SESSION_META_SKIP_REASON: &str = "malformed_session_meta";
+const BUSY_SKIP_REASON: &str = "busy";
const CURSOR_LOOKBACK_SECONDS: i64 = 48 * 60 * 60;
+const MAINTENANCE_RETRY_DELAY: Duration = Duration::from_secs(1);
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct RolloutFingerprint {
size_bytes: i64,
modified_at_ns: i64,
@@ -44,6 +53,7 @@ struct RolloutFingerprint {
enum StartupInspection {
Paginated,
Legacy,
+ NeedsMigration,
Skipped,
Unresolved,
}
@@ -52,8 +62,13 @@ pub(super) async fn migrate_rollouts_on_startup(store: &LocalThreadStore) -> Thr
let Some(state_db) = store.state_db.as_ref() else {
return Ok(());
};
- let paths = find_all_rollout_paths(store).await?;
- let skipped_rollouts = state_db
+ let paths = find_all_rollout_paths(&store.config.codex_home).await?;
+ let mut skipped_rollouts = state_db
+ .list_rollout_migration_skipped_rollouts(LEGACY_TO_PAGINATED_MIGRATION_ID)
+ .await
+ .map_err(migration_error)?;
+ retry_busy_rollouts(store, skipped_rollouts.as_slice(), paths.as_slice()).await?;
+ skipped_rollouts = state_db
.list_rollout_migration_skipped_rollouts(LEGACY_TO_PAGINATED_MIGRATION_ID)
.await
.map_err(migration_error)?;
@@ -63,14 +78,13 @@ pub(super) async fn migrate_rollouts_on_startup(store: &LocalThreadStore) -> Thr
{
return migrate_all_rollouts(store, paths, skipped_rollouts.as_slice()).await;
}
- let (unchanged_skips, invalidated_skip) =
- revalidate_skipped_rollouts(store, skipped_rollouts.as_slice()).await?;
+ let skipped_file_names = skipped_rollout_file_names(store, skipped_rollouts.as_slice());
let state = state_db
.get_rollout_migration_state(LEGACY_TO_PAGINATED_MIGRATION_ID)
.await
.map_err(migration_error)?;
- if state.is_none() || invalidated_skip {
+ if state.is_none() {
return migrate_all_rollouts(store, paths, skipped_rollouts.as_slice()).await;
}
@@ -83,8 +97,8 @@ pub(super) async fn migrate_rollouts_on_startup(store: &LocalThreadStore) -> Thr
let candidates = paths
.iter()
.filter(|path| {
- let relative_path = relative_rollout_path(store, path);
- !unchanged_skips.contains(relative_path.as_str())
+ !plain_rollout_file_name(path)
+ .is_some_and(|file_name| skipped_file_names.contains(&file_name))
&& thread_creation_cursor(path).is_none_or(|cursor| {
lookback_created_at.is_none_or(|lookback_created_at| {
cursor.thread_created_at >= lookback_created_at
@@ -100,7 +114,7 @@ pub(super) async fn migrate_rollouts_on_startup(store: &LocalThreadStore) -> Thr
for path in candidates {
match inspect_rollout_path(store, path).await? {
StartupInspection::Paginated | StartupInspection::Skipped => {}
- StartupInspection::Legacy => {
+ StartupInspection::Legacy | StartupInspection::NeedsMigration => {
return migrate_all_rollouts(store, paths, skipped_rollouts.as_slice()).await;
}
StartupInspection::Unresolved => unresolved = true,
@@ -118,55 +132,132 @@ async fn migrate_all_rollouts(
paths_before_migration: Vec<PathBuf>,
existing_skips: &[RolloutMigrationSkippedRollout],
) -> ThreadStoreResult<()> {
- let report = store
- .migrate_rollouts_with_progress_for_trigger(
- RolloutMigrationOptions {
- mode: RolloutMigrationMode::Apply,
- thread_ids: Vec::new(),
- max_mib_per_second: None,
- },
- |_| {},
- RolloutMigrationTrigger::Startup,
- )
- .await?;
- let existing_skip_paths = existing_skips
+ let skipped_file_names = skipped_rollout_file_names(store, existing_skips);
+ let pending_thread_ids = pending_migration_thread_ids(&store.config.codex_home).await?;
+ let paths_to_migrate = paths_before_migration
.iter()
- .map(|skipped_rollout| skipped_rollout.rollout_path.as_str())
- .collect::<HashSet<_>>();
- let mut terminal = true;
- let mut reported_paths = HashSet::new();
+ .filter(|path| {
+ !plain_rollout_file_name(path)
+ .is_some_and(|file_name| skipped_file_names.contains(&file_name))
+ || thread_id_from_rollout_filename(path)
+ .is_some_and(|thread_id| pending_thread_ids.contains(&thread_id))
+ })
+ .cloned()
+ .collect();
+ let report = run_startup_migration(store, paths_to_migrate).await?;
for outcome in &report.outcomes {
- let relative_path = relative_rollout_path(store, &outcome.rollout_path);
- reported_paths.insert(relative_path.clone());
- match outcome.status {
- RolloutMigrationStatus::Migrated | RolloutMigrationStatus::AlreadyPaginated => {
- if existing_skip_paths.contains(relative_path.as_str()) {
- remove_skip(store, relative_path.as_str()).await?;
- }
- }
- RolloutMigrationStatus::SkippedEmpty | RolloutMigrationStatus::Failed => {
- if !matches!(
- inspect_rollout_path(store, &outcome.rollout_path).await?,
- StartupInspection::Skipped
- ) {
- terminal = false;
- }
- }
- RolloutMigrationStatus::Eligible | RolloutMigrationStatus::SkippedBusy => {
- terminal = false
- }
+ update_skip_after_outcome(store, outcome).await?;
+ }
+ // Only mark the pre-migration snapshot; newer rollouts wait for the next startup check.
+ advance_last_checked_thread(store, paths_before_migration.as_slice()).await
+}
+
+async fn retry_busy_rollouts(
+ store: &LocalThreadStore,
+ skipped_rollouts: &[RolloutMigrationSkippedRollout],
+ discovered_paths: &[PathBuf],
+) -> ThreadStoreResult<()> {
+ let mut paths = Vec::new();
+ let mut moved_skip_paths = Vec::new();
+ for skipped_rollout in skipped_rollouts
+ .iter()
+ .filter(|skipped_rollout| skipped_rollout.skip_reason == BUSY_SKIP_REASON)
+ {
+ let stored_path = store.config.codex_home.join(&skipped_rollout.rollout_path);
+ let path = if tokio::fs::try_exists(&stored_path)
+ .await
+ .map_err(migration_error)?
+ {
+ Some(stored_path.clone())
+ } else {
+ // Archive/unarchive moves one rollout between roots, while compression swaps between
+ // its plain and compressed filenames. Match the plain basename across both.
+ let file_name = plain_rollout_file_name(&stored_path);
+ discovered_paths
+ .iter()
+ .find(|path| plain_rollout_file_name(path) == file_name)
+ .cloned()
+ };
+ let Some(path) = path else {
+ continue;
+ };
+ if path != stored_path {
+ moved_skip_paths.push(skipped_rollout.rollout_path.as_str());
}
+ paths.push(path);
}
- if !terminal {
+ if paths.is_empty() {
return Ok(());
}
- for skipped_rollout in existing_skips {
- if !reported_paths.contains(skipped_rollout.rollout_path.as_str()) {
- remove_skip(store, skipped_rollout.rollout_path.as_str()).await?;
+ let report = run_startup_migration(store, paths).await?;
+ for moved_skip_path in moved_skip_paths {
+ remove_skip(store, moved_skip_path).await?;
+ }
+ for outcome in &report.outcomes {
+ update_skip_after_outcome(store, outcome).await?;
+ }
+ Ok(())
+}
+
+async fn run_startup_migration(
+ store: &LocalThreadStore,
+ paths: Vec<PathBuf>,
+) -> ThreadStoreResult<RolloutMigrationReport> {
+ loop {
+ let Some(maintenance_guard) =
+ codex_rollout::try_acquire_rollout_maintenance_lock(&store.config.codex_home)
+ .map_err(migration_error)?
+ else {
+ tokio::time::sleep(MAINTENANCE_RETRY_DELAY).await;
+ continue;
+ };
+ // Avoid counting expected compression contention as a failed migration run. The migration
+ // path takes the real lock below, so retry if another maintainer wins this small gap.
+ drop(maintenance_guard);
+ match store
+ .migrate_rollouts_with_progress_for_trigger(
+ RolloutMigrationOptions {
+ mode: RolloutMigrationMode::Apply,
+ thread_ids: Vec::new(),
+ max_mib_per_second: None,
+ },
+ |_| {},
+ RolloutMigrationTrigger::Startup,
+ super::RolloutMigrationPaths::Known(paths.clone()),
+ )
+ .await
+ {
+ Err(ThreadStoreError::Conflict { .. }) => continue,
+ result => return result,
}
}
- // Only mark the pre-migration snapshot; newer rollouts wait for the next startup check.
- advance_last_checked_thread(store, paths_before_migration.as_slice()).await
+}
+
+async fn update_skip_after_outcome(
+ store: &LocalThreadStore,
+ outcome: &super::RolloutMigrationOutcome,
+) -> ThreadStoreResult<()> {
+ let relative_path = relative_rollout_path(store, &outcome.rollout_path);
+ match outcome.status {
+ RolloutMigrationStatus::Migrated | RolloutMigrationStatus::AlreadyPaginated => {
+ remove_skip(store, relative_path.as_str()).await
+ }
+ RolloutMigrationStatus::SkippedEmpty => {
+ record_current_skip(store, &outcome.rollout_path, EMPTY_SKIP_REASON).await
+ }
+ RolloutMigrationStatus::SkippedBusy => {
+ record_current_skip(store, &outcome.rollout_path, BUSY_SKIP_REASON).await
+ }
+ RolloutMigrationStatus::Failed => {
+ if outcome.thread_id.is_some_and(|thread_id| {
+ migration_journal_path(&store.config.codex_home, thread_id).exists()
+ }) {
+ return Ok(());
+ }
+ record_current_skip(store, &outcome.rollout_path, FAILED_SKIP_REASON).await
+ }
+ RolloutMigrationStatus::Eligible => Ok(()),
+ }
}
async fn inspect_rollout_path(
@@ -179,33 +270,45 @@ async fn inspect_rollout_path(
Ok(StartupInspection::Legacy)
}
Ok(_) => Ok(StartupInspection::Paginated),
- Err(error) => {
+ Err(_) => {
let after = rollout_fingerprint(path).await?;
- if before != after || !matches!(error.kind(), ErrorKind::Other | ErrorKind::InvalidData)
- {
+ if before != after {
return Ok(StartupInspection::Unresolved);
}
- record_skip(store, path, before).await?;
+ // The migration path re-reads empty files under the writer lock before deciding
+ // whether they are terminally empty or just waiting for SessionMeta.
+ if before.size_bytes == 0 {
+ return Ok(StartupInspection::NeedsMigration);
+ }
+ record_skip(store, path, before, MALFORMED_SESSION_META_SKIP_REASON).await?;
Ok(StartupInspection::Skipped)
}
}
}
+async fn record_current_skip(
+ store: &LocalThreadStore,
+ path: &Path,
+ skip_reason: &str,
+) -> ThreadStoreResult<()> {
+ // These fields remain in the generic schema, but background skips are permanent now. Keep
+ // recording the best available fingerprint for humans inspecting SQLite.
+ let fingerprint = rollout_fingerprint(path).await.unwrap_or_default();
+ record_skip(store, path, fingerprint, skip_reason).await
+}
+
async fn record_skip(
store: &LocalThreadStore,
path: &Path,
fingerprint: RolloutFingerprint,
+ skip_reason: &str,
) -> ThreadStoreResult<()> {
let state_db = startup_state_db(store)?;
let skipped_rollout = RolloutMigrationSkippedRollout {
rollout_path: relative_rollout_path(store, path),
rollout_size_bytes: fingerprint.size_bytes,
rollout_modified_at_ns: fingerprint.modified_at_ns,
- skip_reason: if fingerprint.size_bytes == 0 {
- EMPTY_SKIP_REASON.to_string()
- } else {
- MALFORMED_SESSION_META_SKIP_REASON.to_string()
- },
+ skip_reason: skip_reason.to_string(),
};
state_db
.record_rollout_migration_skip(LEGACY_TO_PAGINATED_MIGRATION_ID, &skipped_rollout)
@@ -220,32 +323,6 @@ async fn remove_skip(store: &LocalThreadStore, rollout_path: &str) -> ThreadStor
.map_err(migration_error)
}
-async fn revalidate_skipped_rollouts(
- store: &LocalThreadStore,
- skipped_rollouts: &[RolloutMigrationSkippedRollout],
-) -> ThreadStoreResult<(HashSet<String>, bool)> {
- let mut unchanged_skips = HashSet::new();
- let mut invalidated_skip = false;
- for skipped_rollout in skipped_rollouts {
- let path = store.config.codex_home.join(&skipped_rollout.rollout_path);
- let fingerprint = match rollout_fingerprint(&path).await {
- Ok(fingerprint) => fingerprint,
- Err(_) => {
- invalidated_skip = true;
- continue;
- }
- };
- if fingerprint.size_bytes == skipped_rollout.rollout_size_bytes
- && fingerprint.modified_at_ns == skipped_rollout.rollout_modified_at_ns
- {
- unchanged_skips.insert(skipped_rollout.rollout_path.clone());
- } else {
- invalidated_skip = true;
- }
- }
- Ok((unchanged_skips, invalidated_skip))
-}
-
async fn advance_last_checked_thread(
store: &LocalThreadStore,
paths: &[PathBuf],
@@ -270,21 +347,6 @@ fn startup_state_db(store: &LocalThreadStore) -> ThreadStoreResult<&StateDbHandl
.ok_or_else(|| migration_error("startup migration requires state db"))
}
-async fn find_all_rollout_paths(store: &LocalThreadStore) -> ThreadStoreResult<Vec<PathBuf>> {
- let mut paths =
- find_rollout_paths(&store.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR)).await?;
- paths.extend(
- find_rollout_paths(
- &store
- .config
- .codex_home
- .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR),
- )
- .await?,
- );
- Ok(paths)
-}
-
fn thread_creation_cursor(path: &Path) -> Option<RolloutMigrationCursor> {
let name = path.file_name()?.to_str()?;
let stem = name
@@ -311,6 +373,24 @@ fn relative_rollout_path(store: &LocalThreadStore, path: &Path) -> String {
.replace('\\', "/")
}
+fn skipped_rollout_file_names(
+ store: &LocalThreadStore,
+ skipped_rollouts: &[RolloutMigrationSkippedRollout],
+) -> HashSet<OsString> {
+ skipped_rollouts
+ .iter()
+ .filter_map(|skipped_rollout| {
+ plain_rollout_file_name(&store.config.codex_home.join(&skipped_rollout.rollout_path))
+ })
+ .collect()
+}
+
+fn plain_rollout_file_name(path: &Path) -> Option<OsString> {
+ codex_rollout::plain_rollout_path(path)
+ .file_name()
+ .map(std::ffi::OsStr::to_os_string)
+}
+
async fn rollout_fingerprint(path: &Path) -> ThreadStoreResult<RolloutFingerprint> {
let metadata = tokio::fs::metadata(path).await.map_err(migration_error)?;
let size_bytes = i64::try_from(metadata.len()).map_err(migration_error)?;
diff --git a/codex-rs/thread-store/src/local/rollout_migration/startup_tests.rs b/codex-rs/thread-store/src/local/rollout_migration/startup_tests.rs
index 041dd1df45..7a4512bfcd 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/startup_tests.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/startup_tests.rs
@@ -199,20 +199,33 @@ async fn checks_rollouts_within_the_cursor_lookback() {
}
#[tokio::test]
-async fn recovers_pending_migrations_behind_the_checked_thread() {
+async fn recovers_pending_migrations_after_retrying_busy_rollouts() {
let home = TempDir::new().expect("create Codex home");
- let thread_id = ThreadId::new();
- let path = write_rollout(home.path(), thread_id, ThreadHistoryMode::Legacy);
+ let pending_thread_id = ThreadId::new();
+ write_rollout(home.path(), pending_thread_id, ThreadHistoryMode::Legacy);
+ let busy_thread_id = ThreadId::new();
+ let busy_path = move_to_timestamp(
+ home.path(),
+ write_rollout(home.path(), busy_thread_id, ThreadHistoryMode::Legacy),
+ "2025/01/04",
+ "2025-01-04T12-00-00",
+ );
let store = indexed_store(home.path()).await;
+ let state_db = store.state_db().await.expect("state db");
+ let writer_guard = store
+ .writer_lock_coordinator
+ .acquire(busy_thread_id)
+ .expect("hold cross-process writer lock");
store
.migrate_rollouts_on_startup()
.await
- .expect("migrate and advance startup cursor");
- thread_history::delete_thread(&store, thread_id)
+ .expect("migrate and record busy rollout");
+ drop(writer_guard);
+ thread_history::delete_thread(&store, pending_thread_id)
.await
.expect("simulate missing projection");
- let journal_path = migration_journal_path(home.path(), thread_id);
+ let journal_path = migration_journal_path(home.path(), pending_thread_id);
write_migration_journal(&journal_path)
.await
.expect("simulate pending migration marker");
@@ -220,22 +233,29 @@ async fn recovers_pending_migrations_behind_the_checked_thread() {
store
.migrate_rollouts_on_startup()
.await
- .expect("recover pending migration behind cursor");
+ .expect("retry busy rollout before recovery");
+ assert_eq!(
+ codex_rollout::read_session_meta_line(&busy_path)
+ .await
+ .expect("read migrated metadata")
+ .meta
+ .history_mode,
+ ThreadHistoryMode::Paginated
+ );
assert!(!journal_path.exists());
assert!(
- thread_history::projection_state(&store, thread_id)
+ thread_history::projection_state(&store, pending_thread_id)
.await
.expect("read repaired projection")
.is_some()
);
- assert_eq!(
- codex_rollout::read_session_meta_line(&path)
+ assert!(
+ state_db
+ .list_rollout_migration_skipped_rollouts(super::LEGACY_TO_PAGINATED_MIGRATION_ID)
.await
- .expect("read migrated metadata")
- .meta
- .history_mode,
- ThreadHistoryMode::Paginated
+ .expect("read skipped rollouts")
+ .is_empty()
);
}
@@ -274,46 +294,180 @@ async fn waits_for_a_live_writer_before_migrating() {
}
#[tokio::test]
-async fn rechecks_changed_empty_rollouts() {
+async fn waits_for_rollout_maintenance_before_migrating() {
let home = TempDir::new().expect("create Codex home");
- write_rollout(home.path(), ThreadId::new(), ThreadHistoryMode::Legacy);
- let empty_thread_id = ThreadId::new();
- let empty_path = move_to_timestamp(
- home.path(),
- write_rollout(home.path(), empty_thread_id, ThreadHistoryMode::Legacy),
- "2025/01/04",
- "2025-01-04T12-00-00",
+ let thread_id = ThreadId::new();
+ let path = write_rollout(home.path(), thread_id, ThreadHistoryMode::Legacy);
+ let store = indexed_store(home.path()).await;
+ let maintenance_guard = codex_rollout::try_acquire_rollout_maintenance_lock(home.path())
+ .expect("acquire rollout maintenance lock")
+ .expect("claim rollout maintenance lock");
+ let migration_store = store.clone();
+ let mut migration = tokio::spawn(async move {
+ migration_store
+ .migrate_rollouts_on_startup()
+ .await
+ .expect("migrate startup rollouts");
+ });
+
+ assert!(
+ tokio::time::timeout(Duration::from_millis(50), &mut migration)
+ .await
+ .is_err(),
+ "migration should wait for rollout maintenance"
);
- let restored_contents = fs::read(&empty_path).expect("read rollout before emptying");
- fs::write(&empty_path, []).expect("empty rollout");
+ drop(maintenance_guard);
+ tokio::time::timeout(Duration::from_secs(2), migration)
+ .await
+ .expect("migration should retry rollout maintenance")
+ .expect("join startup migration");
+
+ assert_eq!(
+ codex_rollout::read_session_meta_line(&path)
+ .await
+ .expect("read migrated metadata")
+ .meta
+ .history_mode,
+ ThreadHistoryMode::Paginated
+ );
+}
+
+#[tokio::test]
+async fn permanently_skips_failed_rollouts_without_blocking_the_cursor() {
+ let home = TempDir::new().expect("create Codex home");
+ let failed_thread_id = ThreadId::new();
+ let failed_path = write_rollout(home.path(), failed_thread_id, ThreadHistoryMode::Legacy);
let store = indexed_store(home.path()).await;
+ let state_db = store.state_db().await.expect("state db");
+ let metadata = state_db
+ .get_thread(failed_thread_id)
+ .await
+ .expect("read thread metadata")
+ .expect("thread metadata");
+ state_db
+ .delete_thread(failed_thread_id)
+ .await
+ .expect("remove thread metadata");
store
.migrate_rollouts_on_startup()
.await
- .expect("record empty rollout");
- fs::write(&empty_path, restored_contents).expect("restore rollout");
- let (items, _, _) = RolloutRecorder::load_rollout_items(&empty_path)
+ .expect("record failed rollout");
+ assert_eq!(
+ state_db
+ .get_rollout_migration_state(super::LEGACY_TO_PAGINATED_MIGRATION_ID)
+ .await
+ .expect("read migration state")
+ .expect("migration state")
+ .last_checked_thread
+ .expect("checked thread")
+ .thread_id,
+ failed_thread_id.to_string()
+ );
+ assert_eq!(
+ state_db
+ .list_rollout_migration_skipped_rollouts(super::LEGACY_TO_PAGINATED_MIGRATION_ID)
+ .await
+ .expect("read skipped rollouts")
+ .into_iter()
+ .map(|skipped_rollout| skipped_rollout.skip_reason)
+ .collect::<Vec<_>>(),
+ vec![super::FAILED_SKIP_REASON.to_string()]
+ );
+
+ state_db
+ .insert_thread_if_absent(&metadata)
.await
- .expect("load restored rollout");
- let metadata = codex_rollout::builder_from_items(items.as_slice(), &empty_path)
- .expect("build restored metadata")
- .build("test-provider");
+ .expect("restore thread metadata");
+ let archived_directory = home.path().join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR);
+ fs::create_dir_all(&archived_directory).expect("create archived directory");
+ let archived_path = archived_directory.join(failed_path.file_name().expect("rollout filename"));
+ fs::rename(&failed_path, &archived_path).expect("archive failed rollout");
+
store
- .state_db()
+ .migrate_rollouts_on_startup()
.await
- .expect("state db")
- .upsert_thread(&metadata)
+ .expect("skip failed rollout again");
+
+ assert_eq!(
+ codex_rollout::read_session_meta_line(&archived_path)
+ .await
+ .expect("read failed rollout metadata")
+ .meta
+ .history_mode,
+ ThreadHistoryMode::Legacy
+ );
+}
+
+#[tokio::test]
+async fn retries_busy_rollouts_after_archive_and_compression_move() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let path = move_to_timestamp(
+ home.path(),
+ write_rollout(home.path(), thread_id, ThreadHistoryMode::Legacy),
+ "2025/01/01",
+ "2025-01-01T12-00-00",
+ );
+ let newest_thread_id = ThreadId::new();
+ move_to_timestamp(
+ home.path(),
+ write_rollout(home.path(), newest_thread_id, ThreadHistoryMode::Paginated),
+ "2025/01/04",
+ "2025-01-04T12-00-00",
+ );
+ let store = indexed_store(home.path()).await;
+ let state_db = store.state_db().await.expect("state db");
+ let writer_guard = store
+ .writer_lock_coordinator
+ .acquire(thread_id)
+ .expect("hold cross-process writer lock");
+
+ store
+ .migrate_rollouts_on_startup()
.await
- .expect("seed restored metadata");
+ .expect("record busy rollout");
+ assert_eq!(
+ state_db
+ .get_rollout_migration_state(super::LEGACY_TO_PAGINATED_MIGRATION_ID)
+ .await
+ .expect("read migration state")
+ .expect("migration state")
+ .last_checked_thread
+ .expect("checked thread")
+ .thread_id,
+ newest_thread_id.to_string()
+ );
+ assert_eq!(
+ state_db
+ .list_rollout_migration_skipped_rollouts(super::LEGACY_TO_PAGINATED_MIGRATION_ID)
+ .await
+ .expect("read skipped rollouts")
+ .into_iter()
+ .map(|skipped_rollout| skipped_rollout.skip_reason)
+ .collect::<Vec<_>>(),
+ vec![super::BUSY_SKIP_REASON.to_string()]
+ );
+ drop(writer_guard);
+ let archived_directory = home.path().join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR);
+ fs::create_dir_all(&archived_directory).expect("create archived directory");
+ let archived_path = archived_directory.join(path.file_name().expect("rollout filename"));
+ fs::rename(&path, &archived_path).expect("archive busy rollout");
+ let compressed_path = archived_path.with_extension("jsonl.zst");
+ let mut input = fs::File::open(&archived_path).expect("open archived rollout");
+ let output = fs::File::create(&compressed_path).expect("create compressed rollout");
+ let mut encoder = zstd::stream::write::Encoder::new(output, 0).expect("create encoder");
+ std::io::copy(&mut input, &mut encoder).expect("compress archived rollout");
+ encoder.finish().expect("finish compressed rollout");
+ fs::remove_file(&archived_path).expect("remove plain archived rollout");
store
.migrate_rollouts_on_startup()
.await
- .expect("migrate changed rollout");
+ .expect("retry no-longer-busy rollout");
assert_eq!(
- codex_rollout::read_session_meta_line(&empty_path)
+ codex_rollout::read_session_meta_line(&compressed_path)
.await
.expect("read migrated metadata")
.meta
@@ -321,13 +475,77 @@ async fn rechecks_changed_empty_rollouts() {
ThreadHistoryMode::Paginated
);
assert!(
- store
- .state_db()
- .await
- .expect("state db")
+ state_db
.list_rollout_migration_skipped_rollouts(super::LEGACY_TO_PAGINATED_MIGRATION_ID)
.await
.expect("read skipped rollouts")
.is_empty()
);
}
+
+#[tokio::test]
+async fn treats_writer_owned_empty_rollouts_as_busy() {
+ let home = TempDir::new().expect("create Codex home");
+ write_rollout(home.path(), ThreadId::new(), ThreadHistoryMode::Paginated);
+ let store = indexed_store(home.path()).await;
+ store
+ .migrate_rollouts_on_startup()
+ .await
+ .expect("seed startup cursor");
+
+ let thread_id = ThreadId::new();
+ let path = move_to_timestamp(
+ home.path(),
+ write_rollout(home.path(), thread_id, ThreadHistoryMode::Legacy),
+ "2025/01/04",
+ "2025-01-04T12-00-00",
+ );
+ let (items, _, _) = RolloutRecorder::load_rollout_items(&path)
+ .await
+ .expect("load rollout items");
+ let metadata = codex_rollout::builder_from_items(items.as_slice(), &path)
+ .expect("build thread metadata")
+ .build("test-provider");
+ let contents = fs::read(&path).expect("read rollout before emptying");
+ fs::write(&path, []).expect("empty rollout");
+ let state_db = store.state_db().await.expect("state db");
+ state_db
+ .upsert_thread(&metadata)
+ .await
+ .expect("seed thread metadata");
+ let writer_guard = store
+ .writer_lock_coordinator
+ .acquire(thread_id)
+ .expect("hold cross-process writer lock");
+
+ store
+ .migrate_rollouts_on_startup()
+ .await
+ .expect("record empty rollout as busy");
+ assert_eq!(
+ state_db
+ .list_rollout_migration_skipped_rollouts(super::LEGACY_TO_PAGINATED_MIGRATION_ID)
+ .await
+ .expect("read skipped rollouts")
+ .into_iter()
+ .map(|skipped_rollout| skipped_rollout.skip_reason)
+ .collect::<Vec<_>>(),
+ vec![super::BUSY_SKIP_REASON.to_string()]
+ );
+
+ fs::write(&path, contents).expect("restore rollout");
+ drop(writer_guard);
+ store
+ .migrate_rollouts_on_startup()
+ .await
+ .expect("retry no-longer-empty rollout");
+
+ assert_eq!(
+ codex_rollout::read_session_meta_line(&path)
+ .await
+ .expect("read migrated metadata")
+ .meta
+ .history_mode,
+ ThreadHistoryMode::Paginated
+ );
+}
diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rs
index 96ee8cb0c2..9e7a380ac1 100644
--- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs
@@ -48,11 +48,13 @@ use super::LocalThreadStore;
use super::RolloutMigrationFailureReason;
use super::RolloutMigrationMode;
use super::RolloutMigrationOptions;
+use super::RolloutMigrationPaths;
use super::RolloutMigrationProgress;
use super::RolloutMigrationStatus;
#[cfg(unix)]
use super::decompress_rollout_to_path;
use super::migration_journal_path;
+use super::telemetry::RolloutMigrationTrigger;
use super::thread_history;
use super::write_migration_journal;
use crate::ItemSortKey;
@@ -1786,6 +1788,37 @@ async fn migration_migrates_archived_rollouts_without_unarchiving_them() {
assert_eq!(turns.turns[0].items.len(), 2);
}
+#[tokio::test]
+async fn migration_retries_a_rollout_moved_after_path_discovery() {
+ let home = TempDir::new().expect("create Codex home");
+ let thread_id = ThreadId::new();
+ let active_path = write_rollout(
+ home.path(),
+ thread_id,
+ SessionSource::Cli,
+ vec![user_message("question"), agent_message("answer")],
+ );
+ let store = indexed_store(home.path()).await;
+ let archived_path = move_to_archived(home.path(), active_path.clone());
+
+ let report = store
+ .migrate_rollouts_with_progress_for_trigger(
+ apply_options(),
+ |_| {},
+ RolloutMigrationTrigger::Startup,
+ RolloutMigrationPaths::Known(vec![active_path]),
+ )
+ .await
+ .expect("migrate moved rollout");
+
+ assert_eq!(report.outcomes[0].status, RolloutMigrationStatus::Migrated);
+ assert!(matches!(
+ &read_rollout(&archived_path)[0].item,
+ RolloutItem::SessionMeta(metadata)
+ if metadata.meta.history_mode == ThreadHistoryMode::Paginated
+ ));
+}
+
#[tokio::test]
async fn migration_preserves_legacy_displayed_thread_names() {
let home = TempDir::new().expect("create Codex home");