Lightweight coding agent that runs in your terminal
Rust
use std::cell::Cell;
use std::fs;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use chrono::Utc;
use codex_app_server_protocol::McpToolCallResult;
use codex_app_server_protocol::ThreadItem;
use codex_protocol::ThreadId;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::AgentMessageItem;
use codex_protocol::items::CommandExecutionItem;
use codex_protocol::items::CommandExecutionStatus;
use codex_protocol::items::McpToolCallItem;
use codex_protocol::items::McpToolCallStatus;
use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::mcp::CallToolResult;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ContentItem;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::ResponseItem;
use codex_protocol::parse_command::ParsedCommand;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ExecCommandSource;
use codex_protocol::protocol::HistoryPosition;
use codex_protocol::protocol::ItemCompletedEvent;
use codex_protocol::protocol::RateLimitSnapshot;
use codex_protocol::protocol::RateLimitWindow;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadMemoryMode;
use codex_protocol::protocol::TokenCountEvent;
use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnStartedEvent;
use codex_protocol::protocol::UserMessageEvent;
use codex_protocol::realtime::BemItemPresentation;
use codex_protocol::realtime::RealtimeItem;
use codex_protocol::realtime::RealtimeItemContent;
use codex_protocol::realtime::RealtimeSessionOutcome;
use codex_protocol::realtime::RealtimeTranscriptRole;
use codex_rollout::RolloutConfig;
use codex_rollout::RolloutItem;
use codex_rollout::RolloutLine;
use codex_rollout::RolloutRecorder;
use codex_rollout::RolloutRecorderParams;
use codex_utils_absolute_path::test_support::PathExt;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
use super::super::LocalThreadStore;
use super::super::LocalThreadStoreConfig;
use super::super::test_support::test_config;
use crate::AppendThreadItemsParams;
use crate::CreateThreadParams;
use crate::DeleteThreadParams;
use crate::ForkBoundary;
use crate::ListThreadsParams;
use crate::ListTurnsParams;
use crate::PersistContext;
use crate::PrepareForkParams;
use crate::PreparedFork;
use crate::ResumeThreadParams;
use crate::SortDirection;
use crate::StoredTurnItemsView;
use crate::StoredTurnStatus;
use crate::ThreadPersistenceMetadata;
use crate::ThreadSortKey;
use crate::ThreadStore;
#[tokio::test]
async fn paginated_history_without_state_db_does_not_initialize_sqlite() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let sqlite = config.sqlite.clone();
let store = LocalThreadStore::new(config, /*state_db*/ None);
let thread_id = ThreadId::default();
assert!(!store.supports_paginated_history_lists());
create_paginated_thread(&store, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
.expect("append paginated rollout");
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist paginated rollout");
store
.flush_thread(thread_id)
.await
.expect("flush paginated rollout");
store
.shutdown_thread(thread_id)
.await
.expect("shutdown paginated rollout");
for runtime_db in sqlite.runtime_db_paths() {
assert!(
!runtime_db.path.exists(),
"expected no SQLite initialization for {}",
runtime_db.path.display()
);
}
}
/// Separate Codex and SQLite homes must work together across startup backfill,
/// thread listing, and projection-backed paginated history reads.
#[tokio::test]
async fn split_homes_support_backfill_listing_and_paginated_history() {
let root = TempDir::new().expect("temp dir");
let codex_home = root.path().join("codex");
let sqlite_home = root.path().join("sqlite");
let thread_id = ThreadId::new();
let sqlite = codex_state::SqliteConfig::new_for_testing(sqlite_home.as_path().abs());
let rollout_config = RolloutConfig {
codex_home: codex_home.clone(),
sqlite: sqlite.clone(),
cwd: codex_home.clone(),
model_provider_id: "test-provider".to_string(),
generate_memories: false,
};
let recorder = RolloutRecorder::new(
&rollout_config,
RolloutRecorderParams::new(
thread_id,
/*forked_from_id*/ None,
/*parent_thread_id*/ None,
SessionSource::Exec,
/*thread_source*/ None,
"test-originator".to_string(),
BaseInstructions::default(),
Vec::new(),
)
.with_history_mode(ThreadHistoryMode::Paginated)
.with_initial_window_id("window-1".to_string()),
)
.await
.expect("create paginated rollout");
recorder
.record_canonical_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
UserMessageEvent {
message: "existing thread".to_string(),
..Default::default()
},
))])
.await
.expect("record existing user message");
recorder.persist().await.expect("persist paginated rollout");
let rollout_path = recorder.rollout_path().to_path_buf();
recorder.shutdown().await.expect("close paginated rollout");
let runtime = codex_rollout::state_db::try_init(&rollout_config)
.await
.expect("backfill state from Codex home");
assert!(
runtime
.get_thread(thread_id)
.await
.expect("read backfilled thread")
.is_some(),
"startup backfill should index the rollout"
);
let store = LocalThreadStore::new(
LocalThreadStoreConfig::from_config(&rollout_config),
Some(runtime),
);
let threads = store
.list_threads(ListThreadsParams {
page_size: 10,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: Vec::new(),
model_providers: None,
cwd_filters: None,
archived: false,
search_term: None,
relation_filter: None,
section: None,
project_id: None,
use_state_db_only: true,
})
.await
.expect("list backfilled threads");
assert_eq!(threads.items.len(), 1);
assert_eq!(
(
threads.items[0].thread_id,
threads.items[0].rollout_path.as_deref(),
threads.items[0].history_mode,
),
(
thread_id,
Some(rollout_path.as_path()),
ThreadHistoryMode::Paginated,
)
);
store
.resume_thread(ResumeThreadParams {
history_revision: None,
thread_id,
rollout_path: Some(rollout_path),
history: None,
include_archived: false,
metadata: ThreadPersistenceMetadata {
cwd: Some(codex_home.clone()),
model_provider: "test-provider".to_string(),
memory_mode: ThreadMemoryMode::Enabled,
},
})
.await
.expect("resume backfilled thread");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started("turn-1"),
completed_item(
thread_id,
"turn-1",
TurnItem::UserMessage(UserMessageItem {
id: "user-1".to_string(),
client_id: None,
content: Vec::new(),
}),
),
completed_item(
thread_id,
"turn-1",
TurnItem::AgentMessage(AgentMessageItem {
id: "agent-1".to_string(),
content: vec![AgentMessageContent::Text {
text: "done".to_string(),
}],
phase: None,
memory_citation: None,
delivery: None,
questions: None,
}),
),
turn_completed("turn-1"),
],
})
.await
.expect("append paginated history");
let turns = store
.list_turns(ListTurnsParams {
thread_id,
include_archived: false,
cursor: None,
page_size: 10,
sort_direction: SortDirection::Asc,
items_view: StoredTurnItemsView::Summary,
})
.await
.expect("list paginated history");
assert_eq!(
turns
.turns
.iter()
.map(|turn| {
(
turn.turn_id.as_str(),
turn.status,
turn.items
.iter()
.map(|item| item.item_id.as_str())
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
vec![(
"turn-1",
StoredTurnStatus::Completed,
vec!["user-1", "agent-1"],
)]
);
let state_db_path = sqlite.state_db_path();
let thread_history_db_path = sqlite.thread_history_db_path();
for sqlite_path in [&state_db_path, &thread_history_db_path] {
assert!(
sqlite_path.exists(),
"expected SQLite database at {}",
sqlite_path.display()
);
let filename = sqlite_path.file_name().expect("SQLite database filename");
assert!(
!codex_home.join(filename).exists(),
"SQLite database should not be created under Codex home"
);
}
}
#[tokio::test]
async fn paginated_live_append_materializes_turn_items_and_state() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started_with_root("turn-1", "root-1"),
completed_item(
thread_id,
"turn-1",
TurnItem::UserMessage(UserMessageItem {
id: "user-1".to_string(),
client_id: None,
content: Vec::new(),
}),
),
completed_item(
thread_id,
"turn-1",
TurnItem::AgentMessage(AgentMessageItem {
id: "agent-1".to_string(),
content: vec![AgentMessageContent::Text {
text: "done".to_string(),
}],
phase: None,
memory_citation: None,
delivery: None,
questions: None,
}),
),
turn_completed("turn-1"),
],
})
.await
.expect("append paginated items");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let (turn_start_byte_offset, _) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 1);
let (_, turn_end_byte_offset) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 4);
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let turn = sqlx::query_as::<
_,
(
i64,
Option<i64>,
Option<i64>,
Option<i64>,
String,
Option<i64>,
Option<i64>,
Option<i64>,
Option<String>,
Option<String>,
Option<String>,
),
>(
r#"
SELECT
rollout_ordinal,
rollout_byte_offset,
rollout_end_ordinal,
rollout_end_byte_offset,
status,
started_at,
completed_at,
duration_ms,
first_user_item_id,
final_agent_item_id,
root_turn_id
FROM thread_turns
WHERE thread_id = ? AND turn_id = ?
"#,
)
.bind(thread_id.to_string())
.bind("turn-1")
.fetch_one(&pool)
.await
.expect("read projected turn");
assert_eq!(
turn,
(
1,
Some(turn_start_byte_offset),
Some(4),
Some(turn_end_byte_offset),
"completed".to_string(),
Some(10),
Some(20),
Some(10_000),
Some("user-1".to_string()),
Some("agent-1".to_string()),
Some("root-1".to_string()),
)
);
let items = sqlx::query_as::<_, (String, i64)>(
r#"
SELECT item_id, rollout_ordinal
FROM thread_items
WHERE thread_id = ?
ORDER BY rollout_ordinal
"#,
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read projected items");
assert_eq!(
items,
vec![("user-1".to_string(), 2), ("agent-1".to_string(), 3)]
);
let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())
.expect("rollout length");
let projection_state = sqlx::query_as::<_, (i64, i64)>(
r#"
SELECT next_rollout_byte_offset, next_rollout_ordinal
FROM thread_history_projection_state
WHERE thread_id = ?
"#,
)
.bind(thread_id.to_string())
.fetch_one(&pool)
.await
.expect("read projection state");
assert_eq!(projection_state, (rollout_len, 5));
}
#[tokio::test]
async fn paginated_command_history_caps_aggregated_output() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
let output = format!("head\n{}\ntail", "x".repeat(128 * 1024));
let command = CommandExecutionItem {
sandbox_type: None,
model_context: None,
id: "exec-1".to_string(),
plugin_id: None,
script_path: None,
process_id: None,
command: vec!["echo".to_string(), "hello".to_string()],
cwd: home.path().abs().into(),
parsed_cmd: vec![ParsedCommand::Unknown {
cmd: "echo hello".to_string(),
}],
source: ExecCommandSource::Agent,
interaction_input: None,
status: CommandExecutionStatus::Completed,
aggregated_output: Some(output),
exit_code: Some(0),
duration: Some(Duration::from_millis(12)),
};
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started("turn-1"),
completed_item(thread_id, "turn-1", TurnItem::CommandExecution(command)),
turn_completed("turn-1"),
],
})
.await
.expect("append command history");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let item_json = sqlx::query_scalar::<_, String>(
"SELECT item_json FROM thread_items WHERE thread_id = ? AND item_id = ?",
)
.bind(thread_id.to_string())
.bind("exec-1")
.fetch_one(&pool)
.await
.expect("read projected command");
let projected: ThreadItem =
serde_json::from_str(&item_json).expect("deserialize projected command");
let ThreadItem::CommandExecution {
aggregated_output: Some(output),
..
} = projected
else {
panic!("expected projected command output");
};
assert_eq!(output.len(), 64 * 1024);
assert!(output.starts_with("head\n"));
assert!(output.ends_with("\ntail"));
assert!(output.contains("command output truncated for persistence"));
}
#[tokio::test]
async fn paginated_mcp_history_caps_large_results() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
let arguments = serde_json::json!({"query": "large result"});
let tool_call = McpToolCallItem {
id: "mcp-1".to_string(),
server: "test".to_string(),
tool: "large_result".to_string(),
arguments: arguments.clone(),
connector_id: None,
mcp_app_resource_uri: None,
mcp_app_ui: None,
link_id: None,
app_name: None,
action_name: None,
plugin_id: None,
read_only_hint: None,
status: McpToolCallStatus::Failed,
result: Some(CallToolResult {
content: vec![serde_json::json!({
"type": "text",
"text": format!("head\n{}\ntail", "x".repeat(200_000)),
})],
structured_content: None,
is_error: Some(true),
meta: None,
}),
error: None,
duration: Some(Duration::from_millis(25)),
};
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started("turn-1"),
completed_item(thread_id, "turn-1", TurnItem::McpToolCall(tool_call)),
turn_completed("turn-1"),
],
})
.await
.expect("append MCP history");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let item_json = sqlx::query_scalar::<_, String>(
"SELECT item_json FROM thread_items WHERE thread_id = ? AND item_id = ?",
)
.bind(thread_id.to_string())
.bind("mcp-1")
.fetch_one(&pool)
.await
.expect("read projected MCP tool call");
let projected: ThreadItem =
serde_json::from_str(&item_json).expect("deserialize projected MCP tool call");
let ThreadItem::McpToolCall {
arguments: projected_arguments,
result: Some(result),
..
} = projected
else {
panic!("expected projected MCP tool call");
};
assert_eq!(projected_arguments, arguments);
let preview = result.content[0]["text"]
.as_str()
.expect("persisted MCP result preview");
assert_eq!(
*result,
McpToolCallResult {
content: vec![serde_json::json!({
"type": "text",
"text": preview,
})],
structured_content: None,
meta: None,
}
);
assert!(preview.len() < 65 * 1024);
assert!(preview.contains("head"));
assert!(preview.contains("chars truncated"));
assert!(preview.contains("tail"));
}
#[tokio::test]
async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist thread metadata");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
RolloutItem::RealtimeItem(RealtimeItem {
id: "voice:started".to_string(),
realtime_session_id: "voice".to_string(),
content: RealtimeItemContent::RealtimeSessionStarted,
}),
RolloutItem::RealtimeItem(RealtimeItem {
id: "voice:transcript:0".to_string(),
realtime_session_id: "voice".to_string(),
content: RealtimeItemContent::TranscriptSegment {
role: RealtimeTranscriptRole::Assistant,
text: "Here is the result.".to_string(),
},
}),
turn_started("turn-1"),
RolloutItem::RealtimeItem(RealtimeItem {
id: "voice:promoted:agent-1".to_string(),
realtime_session_id: "voice".to_string(),
content: RealtimeItemContent::BemItemPromoted {
turn_id: "turn-1".to_string(),
item_id: "agent-1".to_string(),
presentation: BemItemPresentation::InlineMarkdown,
},
}),
completed_item(
thread_id,
"turn-1",
TurnItem::AgentMessage(AgentMessageItem {
id: "agent-1".to_string(),
content: vec![AgentMessageContent::Text {
text: "Result".to_string(),
}],
phase: None,
memory_citation: None,
delivery: None,
questions: None,
}),
),
RolloutItem::RealtimeItem(RealtimeItem {
id: "voice:closed".to_string(),
realtime_session_id: "voice".to_string(),
content: RealtimeItemContent::RealtimeSessionClosed {
outcome: RealtimeSessionOutcome::Ended,
},
}),
],
})
.await
.expect("append interleaved realtime and ordinary rollout items");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let expected_rows = fs::read_to_string(rollout_path.as_path())
.expect("read canonical rollout")
.lines()
.map(|line| codex_rollout::parse_rollout_line(line).expect("parse rollout line"))
.filter_map(|line| match line.item {
RolloutItem::RealtimeItem(item) => Some((
item.id,
i64::try_from(line.ordinal.expect("paginated rollout ordinal"))
.expect("ordinal fits SQLite integer"),
chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str())
.expect("valid rollout timestamp")
.timestamp_millis(),
serde_json::to_value(item.content)
.expect("serialize realtime content")
.get("type")
.and_then(serde_json::Value::as_str)
.expect("realtime content discriminator")
.to_string(),
)),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
expected_rows
.iter()
.map(|(id, ordinal, _, item_type)| (id.as_str(), *ordinal, item_type.as_str()))
.collect::<Vec<_>>(),
vec![
("voice:started", 1, "realtime_session_started"),
("voice:transcript:0", 2, "transcript_segment"),
("voice:promoted:agent-1", 4, "bem_item_promoted"),
("voice:closed", 6, "realtime_session_closed"),
]
);
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let projected_rows = sqlx::query_as::<_, (String, i64, i64, String)>(
"SELECT item_id, rollout_ordinal, created_at_ms, item_type FROM thread_realtime_items WHERE thread_id = ? ORDER BY rollout_ordinal",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read separately projected realtime items");
assert_eq!(projected_rows, expected_rows);
let ordinary_items = sqlx::query_as::<_, (String, String)>(
"SELECT turn_id, item_id FROM thread_items WHERE thread_id = ? ORDER BY rollout_ordinal",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read ordinary turn items");
assert_eq!(
ordinary_items,
vec![("turn-1".to_string(), "agent-1".to_string())]
);
sqlx::query("DELETE FROM thread_history_projection_state WHERE thread_id = ?")
.bind(thread_id.to_string())
.execute(&pool)
.await
.expect("rewind projection checkpoint");
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("replay canonical rollout");
let replayed_rows = sqlx::query_as::<_, (String, i64, i64, String)>(
"SELECT item_id, rollout_ordinal, created_at_ms, item_type FROM thread_realtime_items WHERE thread_id = ? ORDER BY rollout_ordinal",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read replayed realtime items");
assert_eq!(replayed_rows, expected_rows);
let legacy_thread_id = ThreadId::new();
store
.create_thread(CreateThreadParams {
attached_thread: false,
creator_user_id: None,
creator_account_id: None,
session_id: legacy_thread_id.into(),
thread_id: legacy_thread_id,
extra_config: None,
forked_from_id: None,
parent_thread_id: None,
source: SessionSource::Exec,
thread_source: None,
originator: "test_originator".to_string(),
base_instructions: BaseInstructions::default(),
dynamic_tools: Vec::new(),
selected_capability_roots: Vec::new(),
multi_agent_version: None,
history_mode: ThreadHistoryMode::Legacy,
history_base: None,
subagent_history_start_ordinal: None,
initial_window_id: "window-1".to_string(),
runtime_workspace_roots: None,
metadata: ThreadPersistenceMetadata {
cwd: Some(home.path().to_path_buf()),
model_provider: "test-provider".to_string(),
memory_mode: ThreadMemoryMode::Enabled,
},
})
.await
.expect("create legacy thread");
store
.persist_thread(legacy_thread_id, PersistContext::Standard)
.await
.expect("persist legacy thread metadata");
store
.append_items(AppendThreadItemsParams {
thread_id: legacy_thread_id,
items: vec![RolloutItem::RealtimeItem(RealtimeItem {
id: "legacy:started".to_string(),
realtime_session_id: "legacy".to_string(),
content: RealtimeItemContent::RealtimeSessionStarted,
})],
})
.await
.expect("ignore realtime item for legacy rollout");
let legacy_rollout_path = store
.live_rollout_path(legacy_thread_id)
.await
.expect("legacy rollout path");
let (legacy_items, _, _) = RolloutRecorder::load_rollout_items(legacy_rollout_path.as_path())
.await
.expect("load legacy rollout");
assert!(
!legacy_items
.iter()
.any(|item| matches!(item, RolloutItem::RealtimeItem(_)))
);
let legacy_realtime_count = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM thread_realtime_items WHERE thread_id = ?",
)
.bind(legacy_thread_id.to_string())
.fetch_one(&pool)
.await
.expect("count legacy realtime items");
assert_eq!(legacy_realtime_count, 0);
}
#[tokio::test]
async fn referenced_paginated_rollout_projects_inherited_ordinal_range() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let source_id = ThreadId::default();
create_paginated_thread(&store, source_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id: source_id,
items: vec![
turn_started("source-turn"),
user_message("source message"),
turn_completed("source-turn"),
],
})
.await
.expect("append source history");
let source_path = store
.live_rollout_path(source_id)
.await
.expect("source rollout path");
let (_, source_end_byte_offset) =
rollout_line_byte_offsets(source_path.as_path(), /*ordinal*/ 3);
let child_id = ThreadId::default();
let history_base = HistoryPosition {
thread_id: source_id,
end_ordinal_exclusive: 4,
end_byte_offset: u64::try_from(source_end_byte_offset).expect("source byte offset"),
};
create_paginated_subagent_thread(
&store,
child_id,
Some(history_base),
/*subagent_history_start_ordinal*/ None,
)
.await;
store
.persist_thread(child_id, PersistContext::Standard)
.await
.expect("persist child metadata");
assert_eq!(
prepare_paginated_fork(&store, child_id, ForkBoundary::Latest)
.await
.history_base,
Some(history_base)
);
store
.append_items(AppendThreadItemsParams {
thread_id: child_id,
items: vec![
turn_started("child-turn"),
user_message("child message"),
turn_completed("child-turn"),
],
})
.await
.expect("append child history");
let latest_history_base = prepare_paginated_fork(&store, child_id, ForkBoundary::Latest)
.await
.history_base;
for (boundary, expected_base) in [
(ForkBoundary::Latest, latest_history_base),
(
ForkBoundary::ThroughTurn("source-turn".to_string()),
Some(history_base),
),
(
ForkBoundary::BeforeTurn("child-turn".to_string()),
Some(history_base),
),
(
ForkBoundary::ThroughTurn("child-turn".to_string()),
latest_history_base,
),
(ForkBoundary::BeforeTurn("source-turn".to_string()), None),
] {
let prepared = prepare_paginated_fork(&store, child_id, boundary).await;
assert_eq!(prepared.history_base, expected_base);
assert!(matches!(
prepared.model_context.first(),
Some(RolloutItem::SessionMeta(meta)) if meta.meta.id == child_id
));
assert_eq!(
contains_user_message(&prepared.model_context, "source message"),
expected_base.is_some()
);
assert_eq!(
contains_user_message(&prepared.model_context, "child message"),
expected_base == latest_history_base
);
}
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let turn_ordinal = sqlx::query_scalar::<_, i64>(
"SELECT rollout_ordinal FROM thread_turns WHERE thread_id = ? AND turn_id = ?",
)
.bind(child_id.to_string())
.bind("child-turn")
.fetch_one(&pool)
.await
.expect("read child turn ordinal");
let next_ordinal = sqlx::query_scalar::<_, i64>(
"SELECT next_rollout_ordinal FROM thread_history_projection_state WHERE thread_id = ?",
)
.bind(child_id.to_string())
.fetch_one(&pool)
.await
.expect("read child projection ordinal");
assert_eq!((turn_ordinal, next_ordinal), (5, 8));
}
#[tokio::test]
async fn named_fork_boundaries_reject_invisible_and_noncanonical_turns() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let source_id = ThreadId::default();
create_paginated_thread(&store, source_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id: source_id,
items: vec![turn_started("inherited-turn"), user_message("before fork")],
})
.await
.expect("append inherited active turn");
let history_base = prepare_paginated_fork(&store, source_id, ForkBoundary::Latest)
.await
.history_base;
let child_id = ThreadId::default();
create_paginated_subagent_thread(
&store,
child_id,
history_base,
/*subagent_history_start_ordinal*/ None,
)
.await;
store
.persist_thread(child_id, PersistContext::Standard)
.await
.expect("persist child metadata");
store
.append_items(AppendThreadItemsParams {
thread_id: child_id,
items: vec![turn_started("child-turn"), turn_completed("child-turn")],
})
.await
.expect("append child turn after inherited active turn");
let error = store
.prepare_fork(PrepareForkParams {
thread_id: child_id,
boundary: ForkBoundary::ThroughTurn("inherited-turn".to_string()),
})
.await
.expect_err("cannot fork through an inherited active turn");
assert!(matches!(
error,
crate::ThreadStoreError::InvalidRequest { message }
if message == "lastTurnId 'inherited-turn' identifies an in-progress turn"
));
store
.append_items(AppendThreadItemsParams {
thread_id: source_id,
items: vec![
user_message("after fork"),
turn_completed("inherited-turn"),
turn_started("stale-turn"),
turn_started("replacement-turn"),
turn_completed("replacement-turn"),
completed_item(
source_id,
"review-turn",
TurnItem::UserMessage(UserMessageItem {
id: "review-message".to_string(),
client_id: None,
content: Vec::new(),
}),
),
turn_completed("review-turn"),
],
})
.await
.expect("append invisible, stale, and terminal-only turns");
for (thread_id, boundary, expected_error) in [
(
child_id,
ForkBoundary::ThroughTurn("inherited-turn".to_string()),
"fork boundary exceeds inherited source history",
),
(
source_id,
ForkBoundary::ThroughTurn("stale-turn".to_string()),
"lastTurnId 'stale-turn' identifies an in-progress turn",
),
(
source_id,
ForkBoundary::BeforeTurn("review-turn".to_string()),
"turn review-turn does not have a persisted start boundary",
),
] {
let error = store
.prepare_fork(PrepareForkParams {
thread_id,
boundary,
})
.await
.expect_err("reject an invalid fork boundary");
assert!(matches!(
error,
crate::ThreadStoreError::InvalidRequest { message } if message == expected_error
));
}
}
#[tokio::test]
async fn active_turn_stores_only_its_start_position() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
.expect("append active turn");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let (turn_start_byte_offset, _) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 1);
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let turn_position = sqlx::query_as::<_, (Option<i64>, Option<i64>, Option<i64>)>(
"SELECT rollout_byte_offset, rollout_end_ordinal, rollout_end_byte_offset FROM thread_turns WHERE thread_id = ? AND turn_id = ?",
)
.bind(thread_id.to_string())
.bind("turn-1")
.fetch_one(&pool)
.await
.expect("read active turn position");
assert_eq!(turn_position, (Some(turn_start_byte_offset), None, None));
let (latest_byte_offset, latest_ordinal) = projection_state(&pool, thread_id).await;
let prepared = prepare_paginated_fork(&store, thread_id, ForkBoundary::Latest).await;
assert_eq!(
prepared.history_base,
Some(HistoryPosition {
thread_id,
end_ordinal_exclusive: u64::try_from(latest_ordinal).expect("latest ordinal"),
end_byte_offset: u64::try_from(latest_byte_offset).expect("latest byte offset"),
})
);
assert!(prepared.model_context.iter().any(|item| {
matches!(item, RolloutItem::EventMsg(EventMsg::TurnStarted(event)) if event.turn_id == "turn-1")
}));
// Another store may fork the persisted prefix while this store keeps the source writer open.
let other_store = projection_store(home.path()).await;
let other_prepared =
prepare_paginated_fork(&other_store, thread_id, ForkBoundary::Latest).await;
assert_eq!(
serde_json::to_value((
other_prepared.history_base,
other_prepared.model_context.as_ref()
))
.expect("serialize other store's fork snapshot"),
serde_json::to_value((prepared.history_base, prepared.model_context.as_ref()))
.expect("serialize live store's fork snapshot"),
);
assert_eq!(
prepare_paginated_fork(
&store,
thread_id,
ForkBoundary::BeforeTurn("turn-1".to_string()),
)
.await
.history_base,
None
);
}
#[tokio::test]
async fn paginated_fork_persists_empty_source() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
assert!(!rollout_path.exists());
let prepared = prepare_paginated_fork(&store, thread_id, ForkBoundary::Latest).await;
assert!(rollout_path.exists());
assert_eq!(prepared.history_base, None);
assert!(matches!(
prepared.model_context.as_slice(),
[RolloutItem::SessionMeta(meta)] if meta.meta.id == thread_id
));
}
#[tokio::test]
async fn paginated_fork_reads_compressed_shared_lineage_without_materializing() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let ancestor_thread_id = ThreadId::default();
create_paginated_thread(&store, ancestor_thread_id).await;
store
.persist_thread(ancestor_thread_id, PersistContext::Standard)
.await
.expect("persist ancestor meta");
store
.append_items(AppendThreadItemsParams {
thread_id: ancestor_thread_id,
items: vec![
turn_started("ancestor-turn"),
user_message("inherited ancestor message"),
turn_completed("ancestor-turn"),
],
})
.await
.expect("append ancestor turn");
let ancestor_path = store
.live_rollout_path(ancestor_thread_id)
.await
.expect("ancestor rollout path");
let ancestor_base = prepare_paginated_fork(&store, ancestor_thread_id, ForkBoundary::Latest)
.await
.history_base
.expect("ancestor prefix");
store
.shutdown_thread(ancestor_thread_id)
.await
.expect("shutdown ancestor");
// A standalone source still becomes plain before its first shared reference, so the default
// mode does not introduce compressed lineages that older readers cannot follow.
compress_rollout(ancestor_path.as_path());
assert_eq!(
prepare_paginated_fork(&store, ancestor_thread_id, ForkBoundary::Latest)
.await
.history_base,
Some(ancestor_base)
);
assert!(ancestor_path.exists());
let source_thread_id = ThreadId::default();
create_paginated_subagent_thread(
&store,
source_thread_id,
Some(ancestor_base),
/*subagent_history_start_ordinal*/ None,
)
.await;
store
.persist_thread(source_thread_id, PersistContext::Standard)
.await
.expect("persist source meta");
store
.append_items(AppendThreadItemsParams {
thread_id: source_thread_id,
items: vec![
turn_started("source-turn"),
user_message("inherited source message"),
turn_completed("source-turn"),
],
})
.await
.expect("append source turn");
let source_path = store
.live_rollout_path(source_thread_id)
.await
.expect("source rollout path");
store
.shutdown_thread(source_thread_id)
.await
.expect("shutdown source");
let ancestor_compressed_path = ancestor_path.with_extension("jsonl.zst");
let source_compressed_path = source_path.with_extension("jsonl.zst");
compress_rollout(ancestor_path.as_path());
compress_rollout(source_path.as_path());
let ancestor_modified = fs::metadata(&ancestor_compressed_path)
.and_then(|metadata| metadata.modified())
.expect("read compressed ancestor timestamp");
let source_modified = fs::metadata(&source_compressed_path)
.and_then(|metadata| metadata.modified())
.expect("read compressed source timestamp");
let (first, second) = tokio::join!(
prepare_paginated_fork(&store, source_thread_id, ForkBoundary::Latest),
prepare_paginated_fork(&store, source_thread_id, ForkBoundary::Latest),
);
assert!(!ancestor_path.exists());
assert!(!source_path.exists());
assert!(ancestor_compressed_path.exists());
assert!(source_compressed_path.exists());
assert_eq!(
fs::metadata(&ancestor_compressed_path)
.and_then(|metadata| metadata.modified())
.expect("read compressed ancestor timestamp"),
ancestor_modified
);
assert_eq!(
fs::metadata(&source_compressed_path)
.and_then(|metadata| metadata.modified())
.expect("read compressed source timestamp"),
source_modified
);
for prepared in [first, second] {
assert!(matches!(
prepared.model_context.first(),
Some(RolloutItem::SessionMeta(meta)) if meta.meta.id == source_thread_id
));
for message in ["inherited ancestor message", "inherited source message"] {
assert!(contains_user_message(
prepared.model_context.as_slice(),
message
));
}
}
// Skipping materialization must not allow a reference that cannot be resolved inside this home.
let external_home = TempDir::new().expect("external temp dir");
let external_path = external_home.path().join(
source_compressed_path
.file_name()
.expect("source rollout filename"),
);
fs::rename(&source_compressed_path, &external_path).expect("move shared source outside home");
store
.resume_thread(ResumeThreadParams {
history_revision: None,
thread_id: source_thread_id,
rollout_path: Some(external_path),
history: None,
include_archived: true,
metadata: ThreadPersistenceMetadata {
cwd: Some(home.path().to_path_buf()),
model_provider: "test-provider".to_string(),
memory_mode: ThreadMemoryMode::Enabled,
},
})
.await
.expect("resume external shared source");
let error = store
.prepare_fork(PrepareForkParams {
thread_id: source_thread_id,
boundary: ForkBoundary::Latest,
})
.await
.expect_err("external shared source cannot be referenced by rollout id");
assert!(matches!(
error,
crate::ThreadStoreError::InvalidRequest { message } if message.contains("must be in Codex home")
));
store
.shutdown_thread(source_thread_id)
.await
.expect("shutdown external source");
}
#[tokio::test]
async fn cancelled_fork_keeps_source_reserved_until_lineage_resolution_finishes() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let ancestor_thread_id = ThreadId::default();
create_paginated_thread(&store, ancestor_thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id: ancestor_thread_id,
items: vec![
turn_started("ancestor-turn"),
turn_completed("ancestor-turn"),
],
})
.await
.expect("append ancestor history");
let ancestor_base = prepare_paginated_fork(&store, ancestor_thread_id, ForkBoundary::Latest)
.await
.history_base
.expect("ancestor prefix");
let source_thread_id = ThreadId::default();
create_paginated_subagent_thread(
&store,
source_thread_id,
Some(ancestor_base),
/*subagent_history_start_ordinal*/ None,
)
.await;
store
.persist_thread(source_thread_id, PersistContext::Standard)
.await
.expect("persist source metadata");
let source_path = store
.live_rollout_path(source_thread_id)
.await
.expect("source rollout path");
store
.shutdown_thread(source_thread_id)
.await
.expect("shutdown source");
compress_rollout(source_path.as_path());
let ancestor_writer_guard = store.live_writer_locks.lock(ancestor_thread_id).await;
let source_coordination = store.live_writer_locks.coordination(source_thread_id).await;
let preparation_store = store.clone();
let preparation = tokio::spawn(async move {
preparation_store
.prepare_fork(PrepareForkParams {
thread_id: source_thread_id,
boundary: ForkBoundary::Latest,
})
.await
});
tokio::time::timeout(Duration::from_secs(10), async {
while source_coordination.lifecycle.try_write().is_ok() {
tokio::task::yield_now().await;
}
})
.await
.expect("fork preparation should reserve the source");
preparation.abort();
assert!(
preparation
.await
.expect_err("fork preparation should be cancelled")
.is_cancelled()
);
let mut delete = Box::pin(store.delete_thread(DeleteThreadParams {
thread_id: source_thread_id,
}));
tokio::select! {
biased;
result = &mut delete => {
panic!("source deletion completed while lineage resolution was active: {result:?}")
}
_ = tokio::task::yield_now() => {}
}
drop(ancestor_writer_guard);
delete
.await
.expect("delete source after lineage resolution finishes");
}
#[tokio::test]
async fn prepared_fork_reserves_source_until_child_reference_is_durable() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let source_thread_id = ThreadId::default();
create_paginated_thread(&store, source_thread_id).await;
store
.persist_thread(source_thread_id, PersistContext::Standard)
.await
.expect("persist source metadata");
store
.append_items(AppendThreadItemsParams {
thread_id: source_thread_id,
items: vec![turn_started("source-turn"), turn_completed("source-turn")],
})
.await
.expect("append source turn");
let prepared = store
.prepare_fork(PrepareForkParams {
thread_id: source_thread_id,
boundary: ForkBoundary::Latest,
})
.await
.expect("prepare referenced fork");
let history_base = prepared.history_base.expect("source history base");
let mut delete = Box::pin(store.delete_thread(DeleteThreadParams {
thread_id: source_thread_id,
}));
tokio::select! {
biased;
result = &mut delete => {
panic!("source deletion completed before its child reference was durable: {result:?}")
}
_ = tokio::task::yield_now() => {}
}
tokio::time::timeout(
Duration::from_secs(10),
store.append_items(AppendThreadItemsParams {
thread_id: source_thread_id,
items: vec![
turn_started("later-turn"),
user_message("later source message"),
turn_completed("later-turn"),
],
}),
)
.await
.expect("source write should not wait behind deletion")
.expect("source writes remain available during fork preparation");
assert_eq!(prepared.history_base, Some(history_base));
assert!(!contains_user_message(
prepared.model_context.as_slice(),
"later source message"
));
let child_thread_id = ThreadId::default();
create_paginated_subagent_thread(
&store,
child_thread_id,
Some(history_base),
/*subagent_history_start_ordinal*/ None,
)
.await;
store
.persist_thread(child_thread_id, PersistContext::Standard)
.await
.expect("persist child history reference");
drop(prepared);
let error = delete
.await
.expect_err("durable child reference protects its source");
assert!(
error
.to_string()
.contains("forked history still references")
);
}
#[tokio::test]
async fn subagent_prefix_advances_projection_without_materializing_history() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_subagent_thread(
&store,
thread_id,
/*history_base*/ None,
/*subagent_history_start_ordinal*/ Some(5),
)
.await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
RolloutItem::RealtimeItem(RealtimeItem {
id: "parent:started".to_string(),
realtime_session_id: "parent".to_string(),
content: RealtimeItemContent::RealtimeSessionStarted,
}),
turn_started("parent-turn"),
completed_item(
thread_id,
"parent-turn",
TurnItem::UserMessage(UserMessageItem {
id: "parent-user".to_string(),
client_id: None,
content: Vec::new(),
}),
),
turn_completed("parent-turn"),
turn_started("child-turn"),
RolloutItem::RealtimeItem(RealtimeItem {
id: "child:started".to_string(),
realtime_session_id: "child".to_string(),
content: RealtimeItemContent::RealtimeSessionStarted,
}),
completed_item(
thread_id,
"child-turn",
TurnItem::UserMessage(UserMessageItem {
id: "child-user".to_string(),
client_id: None,
content: Vec::new(),
}),
),
turn_completed("child-turn"),
],
})
.await
.expect("append inherited prefix and child history");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let (child_start_byte_offset, _) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 5);
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let turns = sqlx::query_as::<_, (String, i64, Option<i64>)>(
"SELECT turn_id, rollout_ordinal, rollout_byte_offset FROM thread_turns WHERE thread_id = ?",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read projected turns");
assert_eq!(
turns,
vec![("child-turn".to_string(), 5, Some(child_start_byte_offset))]
);
let items = sqlx::query_as::<_, (String, i64)>(
"SELECT item_id, rollout_ordinal FROM thread_items WHERE thread_id = ?",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read projected items");
assert_eq!(items, vec![("child-user".to_string(), 7)]);
let realtime_items = sqlx::query_as::<_, (String, i64)>(
"SELECT item_id, rollout_ordinal FROM thread_realtime_items WHERE thread_id = ?",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read projected realtime items");
assert_eq!(realtime_items, vec![("child:started".to_string(), 6)]);
assert_eq!(projection_state(&pool, thread_id).await.1, 9);
}
#[tokio::test]
async fn projection_preserves_exact_lifecycle_timestamps() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
let timestamps = [
(Some(1_789_855_978_123), Some(1_789_855_979_456)),
(Some(1_789_855_978_123), Some(1_789_855_978_123)),
(None, Some(1_789_855_979_456)),
(None, None),
];
let mut items = vec![turn_started("turn-1")];
for (index, (started_at_ms, completed_at_ms)) in timestamps.iter().enumerate() {
items.push(RolloutItem::EventMsg(EventMsg::ItemCompleted(
ItemCompletedEvent {
thread_id,
turn_id: "turn-1".to_string(),
item: TurnItem::UserMessage(UserMessageItem {
id: format!("user-{index}"),
client_id: None,
content: Vec::new(),
}),
started_at_ms: *started_at_ms,
completed_at_ms: completed_at_ms.unwrap_or_default(),
},
)));
}
store
.append_items(AppendThreadItemsParams { thread_id, items })
.await
.expect("append lifecycle records");
store
.shutdown_thread(thread_id)
.await
.expect("shutdown thread");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let actual = sqlx::query_as::<_, (Option<i64>, Option<i64>)>(
"SELECT started_at_ms, completed_at_ms FROM thread_items WHERE thread_id = ? ORDER BY rollout_ordinal",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read persisted lifecycle timestamps");
assert_eq!(actual, timestamps);
}
#[tokio::test]
async fn unexpected_duplicate_item_completion_does_not_poison_projection() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started("turn-1"),
completed_item(
thread_id,
"turn-1",
TurnItem::UserMessage(UserMessageItem {
id: "user-1".to_string(),
client_id: None,
content: Vec::new(),
}),
),
],
})
.await
.expect("append completed item");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let first_created_at_ms = sqlx::query_scalar::<_, i64>(
"SELECT created_at_ms FROM thread_items WHERE thread_id = ? AND turn_id = ? AND item_id = ?",
)
.bind(thread_id.to_string())
.bind("turn-1")
.bind("user-1")
.fetch_one(&pool)
.await
.expect("read first item timestamp");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![completed_item(
thread_id,
"turn-1",
TurnItem::UserMessage(UserMessageItem {
id: "user-1".to_string(),
client_id: Some("updated".to_string()),
content: Vec::new(),
}),
)],
})
.await
.expect("append unexpected duplicate item completion");
let item = sqlx::query_as::<_, (i64, i64, i64, String)>(
r#"
SELECT rollout_ordinal, updated_at_ordinal, created_at_ms, item_json
FROM thread_items
WHERE thread_id = ? AND turn_id = ? AND item_id = ?
"#,
)
.bind(thread_id.to_string())
.bind("turn-1")
.bind("user-1")
.fetch_one(&pool)
.await
.expect("read projected item");
assert_eq!((item.0, item.1, item.2), (2, 3, first_created_at_ms));
assert_eq!(
serde_json::from_str::<ThreadItem>(item.3.as_str()).expect("parse projected item"),
ThreadItem::UserMessage {
id: "user-1".to_string(),
client_id: Some("updated".to_string()),
content: Vec::new(),
}
);
}
#[tokio::test]
async fn terminal_turn_does_not_change_after_later_records() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started_with_root("turn-1", "root-1"),
turn_completed("turn-1"),
],
})
.await
.expect("append terminal turn");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started_with_root("turn-1", "wrong-root"),
completed_item(
thread_id,
"turn-1",
TurnItem::UserMessage(UserMessageItem {
id: "late-user".to_string(),
client_id: None,
content: Vec::new(),
}),
),
],
})
.await
.expect("append later records");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let (turn_start_byte_offset, _) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 1);
let (_, turn_end_byte_offset) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 2);
let turn = sqlx::query_as::<
_,
(
i64,
Option<i64>,
Option<i64>,
Option<i64>,
String,
Option<String>,
Option<String>,
),
>(
r#"
SELECT
rollout_ordinal,
rollout_byte_offset,
rollout_end_ordinal,
rollout_end_byte_offset,
status,
first_user_item_id,
root_turn_id
FROM thread_turns
WHERE thread_id = ? AND turn_id = ?
"#,
)
.bind(thread_id.to_string())
.bind("turn-1")
.fetch_one(&pool)
.await
.expect("read projected turn");
assert_eq!(
turn,
(
1,
Some(turn_start_byte_offset),
Some(2),
Some(turn_end_byte_offset),
"completed".to_string(),
None,
Some("root-1".to_string()),
)
);
}
#[tokio::test]
async fn summary_items_use_final_answers_and_ignore_nonterminal_messages() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let thread_id = ThreadId::default();
let runtime = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await
.expect("state runtime");
let mut builder = codex_state::ThreadMetadataBuilder::new(
thread_id,
home.path().join("missing-rollout.jsonl"),
Utc::now(),
SessionSource::Cli,
);
builder.history_mode = ThreadHistoryMode::Paginated;
runtime
.upsert_thread(&builder.build(config.default_model_provider_id.as_str()))
.await
.expect("seed thread metadata");
let store = LocalThreadStore::new(config, Some(runtime));
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
completed_item(
thread_id,
"turn-1",
TurnItem::UserMessage(UserMessageItem {
id: "user-1".to_string(),
client_id: None,
content: Vec::new(),
}),
),
completed_item(
thread_id,
"turn-1",
agent_message("commentary-1", MessagePhase::Commentary),
),
completed_item(
thread_id,
"turn-1",
agent_message("partial-1", MessagePhase::PartialAnswer),
),
completed_item(
thread_id,
"turn-1",
agent_message("final-1", MessagePhase::FinalAnswer),
),
],
})
.await
.expect("append items before turn lifecycle");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
sqlx::query(
r#"
INSERT OR REPLACE INTO thread_items (
thread_id,
turn_id,
item_id,
rollout_ordinal,
created_at_ms,
item_json
)
SELECT thread_id, turn_id, item_id, rollout_ordinal, created_at_ms, item_json
FROM thread_items
WHERE thread_id = ? AND turn_id = ?
"#,
)
.bind(thread_id.to_string())
.bind("turn-1")
.execute(&pool)
.await
.expect("older writers can append items without a stored item type");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1"), turn_completed("turn-1")],
})
.await
.expect("append delayed turn lifecycle");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started("turn-2"),
completed_item(
thread_id,
"turn-2",
TurnItem::UserMessage(UserMessageItem {
id: "user-2".to_string(),
client_id: None,
content: Vec::new(),
}),
),
completed_item(
thread_id,
"turn-2",
agent_message("commentary-2", MessagePhase::Commentary),
),
completed_item(
thread_id,
"turn-2",
agent_message("partial-2", MessagePhase::PartialAnswer),
),
turn_completed("turn-2"),
],
})
.await
.expect("append turn without a final answer");
let summary = store
.list_turns(ListTurnsParams {
thread_id,
include_archived: false,
cursor: None,
page_size: 2,
sort_direction: SortDirection::Asc,
items_view: StoredTurnItemsView::Summary,
})
.await
.expect("list turn summaries");
assert_eq!(
summary
.turns
.iter()
.map(|turn| {
(
turn.turn_id.as_str(),
turn.items
.iter()
.map(|item| item.item_id.as_str())
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
vec![
("turn-1", vec!["user-1", "final-1"]),
("turn-2", vec!["user-2"]),
]
);
}
#[tokio::test]
async fn paginated_projection_accepts_float_rate_limits_and_later_final_answers() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
.expect("append projected turn start");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let checkpoint = projection_state(&pool, thread_id).await;
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let token_count = |primary: Option<f64>, secondary: Option<f64>| {
RolloutItem::EventMsg(EventMsg::TokenCount(TokenCountEvent {
info: None,
rate_limits: Some(RateLimitSnapshot {
limit_id: None,
limit_name: None,
normal_model_slug: None,
primary: primary.map(|used_percent| RateLimitWindow {
used_percent,
window_minutes: Some(60),
resets_at: Some(1_800_000_000),
}),
secondary: secondary.map(|used_percent| RateLimitWindow {
used_percent,
window_minutes: Some(10_080),
resets_at: Some(1_800_100_000),
}),
credits: None,
individual_limit: None,
spend_control_reached: None,
plan_type: None,
rate_limit_reached_type: None,
}),
}))
};
let recorder = store
.live_recorders
.lock()
.await
.get(&thread_id)
.expect("live recorder")
.recorder
.clone();
recorder
.record_canonical_items(&[
token_count(Some(0.0), Some(1.0)),
completed_item(
thread_id,
"turn-1",
agent_message("final-1", MessagePhase::FinalAnswer),
),
token_count(Some(12.5), None),
turn_completed("turn-1"),
])
.await
.expect("queue unprojected history");
recorder.flush().await.expect("flush unprojected history");
assert_eq!(projection_state(&pool, thread_id).await, checkpoint);
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("project floating-point rate limits");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![
turn_started("turn-2"),
token_count(None, Some(87.25)),
completed_item(
thread_id,
"turn-2",
agent_message("final-2", MessagePhase::FinalAnswer),
),
token_count(Some(1e-12), Some(1_000_000.0)),
turn_completed("turn-2"),
],
})
.await
.expect("append history after unprojected floating-point rate limits");
let final_answers = sqlx::query_as::<_, (String, String, String, i64)>(
r#"
SELECT turns.turn_id, items.item_id, turns.status, items.rollout_ordinal
FROM thread_turns AS turns
JOIN thread_items AS items
ON items.thread_id = turns.thread_id
AND items.turn_id = turns.turn_id
AND items.item_id = turns.final_agent_item_id
WHERE turns.thread_id = ?
ORDER BY turns.rollout_ordinal
"#,
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read projected final answers");
assert_eq!(
final_answers,
vec![
(
"turn-1".to_string(),
"final-1".to_string(),
"completed".to_string(),
3,
),
(
"turn-2".to_string(),
"final-2".to_string(),
"completed".to_string(),
8,
),
]
);
let rollout_len = i64::try_from(
fs::metadata(rollout_path.as_path())
.expect("rollout metadata")
.len(),
)
.expect("rollout length");
assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 11));
}
#[tokio::test]
async fn next_write_catches_up_unprojected_durable_suffix() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let checkpoint = projection_state(&pool, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
.expect("append turn start");
let thread_id_string = thread_id.to_string();
sqlx::query("DELETE FROM thread_turns WHERE thread_id = ?")
.bind(thread_id_string.as_str())
.execute(&pool)
.await
.expect("remove projected turn");
sqlx::query(
r#"
UPDATE thread_history_projection_state
SET next_rollout_byte_offset = ?, next_rollout_ordinal = ?
WHERE thread_id = ?
"#,
)
.bind(checkpoint.0)
.bind(checkpoint.1)
.bind(thread_id_string.as_str())
.execute(&pool)
.await
.expect("rewind projection state");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![completed_item(
thread_id,
"turn-1",
TurnItem::UserMessage(UserMessageItem {
id: "user-1".to_string(),
client_id: None,
content: Vec::new(),
}),
)],
})
.await
.expect("append after simulated projection failure");
let rows = sqlx::query_as::<_, (String, String)>(
r#"
SELECT
(SELECT status FROM thread_turns WHERE thread_id = ? AND turn_id = 'turn-1'),
(SELECT item_id FROM thread_items WHERE thread_id = ? AND turn_id = 'turn-1')
"#,
)
.bind(thread_id_string.as_str())
.bind(thread_id_string.as_str())
.fetch_one(&pool)
.await
.expect("read recovered rows");
assert_eq!(rows, ("inProgress".to_string(), "user-1".to_string()));
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())
.expect("rollout length");
assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 3));
}
#[tokio::test]
async fn synchronized_catch_up_does_not_replay_old_rows() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
.expect("append turn start");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let before = projection_state(&pool, thread_id).await;
sqlx::query("UPDATE thread_turns SET status = 'sentinel' WHERE thread_id = ?")
.bind(thread_id.to_string())
.execute(&pool)
.await
.expect("mark projected turn");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("catch up synchronized rollout");
assert_eq!(projection_state(&pool, thread_id).await, before);
let status =
sqlx::query_scalar::<_, String>("SELECT status FROM thread_turns WHERE thread_id = ?")
.bind(thread_id.to_string())
.fetch_one(&pool)
.await
.expect("read projected turn");
assert_eq!(status, "sentinel");
}
/// Large records keep their byte positions and remain pending until their newline is durable.
#[tokio::test]
async fn catch_up_preserves_trailing_partial_line_boundaries() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let before = projection_state(&pool, thread_id).await;
let padding = " ".repeat(64 * 1024);
let complete_line = format!("{}{padding}", rollout_line(Some(1), turn_started("turn-1")));
let partial_line = format!(
"{}{padding}",
rollout_line(Some(2), turn_completed("turn-1"))
);
let complete_suffix = format!("{complete_line}\n");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
append_suffix(
rollout_path.as_path(),
format!("{complete_suffix}{partial_line}").as_str(),
);
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("catch up complete suffix");
let expected_offset =
before.0 + i64::try_from(complete_suffix.len()).expect("complete suffix byte count");
assert_eq!(
projection_state(&pool, thread_id).await,
(expected_offset, 2)
);
append_suffix(rollout_path.as_path(), "\n");
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("catch up completed partial suffix");
let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())
.expect("rollout length");
let turn_position = sqlx::query_as::<_, (Option<i64>, Option<i64>, Option<i64>)>(
"SELECT rollout_byte_offset, rollout_end_ordinal, rollout_end_byte_offset FROM thread_turns WHERE thread_id = ? AND turn_id = ?",
)
.bind(thread_id.to_string())
.bind("turn-1")
.fetch_one(&pool)
.await
.expect("read completed turn position");
assert_eq!(turn_position, (Some(before.0), Some(2), Some(rollout_len)));
}
#[tokio::test]
async fn catch_up_skips_invalid_complete_suffixes_and_projects_later_history() {
let cases = [
(
"missing ordinal",
format!(
"{}\n",
rollout_line(/*ordinal*/ None, turn_started("turn-1"))
),
1,
Vec::new(),
),
(
"duplicate ordinal",
format!(
"{}\n{}\n{}\n",
rollout_line(Some(1), turn_started("turn-1")),
rollout_line(Some(1), turn_started("skipped-turn")),
rollout_line(Some(2), turn_started("turn-2")),
),
3,
vec!["turn-1", "turn-2"],
),
(
"forward ordinal gap",
format!("{}\n", rollout_line(Some(2), turn_started("turn-1"))),
3,
vec!["turn-1"],
),
];
for (name, suffix, expected_next_ordinal, expected_turn_ids) in cases {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let pool = codex_state::open_thread_history_db(
&codex_state::SqliteConfig::new_for_testing(home.path().abs()),
)
.await
.expect("open thread history db");
let before = projection_state(&pool, thread_id).await;
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
append_suffix(rollout_path.as_path(), suffix.as_str());
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect(name);
let expected_offset = before.0 + i64::try_from(suffix.len()).expect("suffix byte count");
assert_eq!(
projection_state(&pool, thread_id).await,
(expected_offset, expected_next_ordinal),
"{name} should advance projection state"
);
let turn_ids = sqlx::query_scalar::<_, String>(
"SELECT turn_id FROM thread_turns WHERE thread_id = ? ORDER BY rollout_ordinal",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read projected turns");
assert_eq!(turn_ids, expected_turn_ids, "{name}");
}
}
#[tokio::test]
async fn jsonl_failure_does_not_create_projection_database() {
let home = TempDir::new().expect("temp dir");
fs::write(home.path().join("sessions"), "not a directory").expect("block sessions dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
.expect_err("JSONL append should fail");
assert!(
!codex_state::SqliteConfig::new_for_testing(home.path().abs())
.thread_history_db_path()
.exists()
);
}
/// A failed batch rolls back atomically; retries start after the last committed complete record.
#[tokio::test]
async fn catch_up_preserves_committed_batches_on_reader_or_writer_failure() {
enum Failure {
Reader,
Writer,
}
for failure in [Failure::Reader, Failure::Writer] {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let pool = codex_state::open_thread_history_db(
&codex_state::SqliteConfig::new_for_testing(home.path().abs()),
)
.await
.expect("open thread history db");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
// More than the producer, queue and consumer can hold: failures exercise backpressure.
append_suffix(rollout_path.as_path(), "\nnot-json\n");
for index in 1..=2049 {
// The gap and following record cross the step limit, but must commit together.
let ordinal = if index >= 256 { index + 1 } else { index };
append_suffix(
rollout_path.as_path(),
&format!(
"{}\n",
rollout_line(Some(ordinal), turn_started(&format!("turn-{index}")))
),
);
}
append_suffix(rollout_path.as_path(), "\n");
let valid_len = fs::metadata(rollout_path.as_path())
.expect("rollout metadata")
.len();
match failure {
Failure::Reader => append_suffix(
rollout_path.as_path(),
&format!(
"{}\n",
rollout_line(Some(u64::MAX), turn_started("overflow"))
),
),
Failure::Writer => {
sqlx::query(
"CREATE TRIGGER reject_projection BEFORE INSERT ON thread_turns
WHEN NEW.turn_id = 'turn-300'
BEGIN SELECT RAISE(FAIL, 'reject projection'); END",
)
.execute(&pool)
.await
.expect("reject a later batch");
}
}
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect_err("failed batch must not commit");
let turn_count =
sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM thread_turns WHERE thread_id = ?")
.bind(thread_id.to_string())
.fetch_one(&pool)
.await
.expect("count committed turns");
let committed_turns = match failure {
Failure::Reader => 2048,
Failure::Writer => 256,
};
let committed_ordinal = committed_turns + 1;
let (_, committed_offset) =
rollout_line_byte_offsets(rollout_path.as_path(), committed_ordinal);
assert_eq!(
(projection_state(&pool, thread_id).await, turn_count),
(
(committed_offset, committed_ordinal as i64 + 1),
committed_turns as i64
)
);
match failure {
Failure::Reader => fs::OpenOptions::new()
.write(true)
.open(rollout_path.as_path())
.expect("open rollout")
.set_len(valid_len)
.expect("remove invalid suffix"),
Failure::Writer => {
sqlx::query("DROP TRIGGER reject_projection")
.execute(&pool)
.await
.expect("allow retry");
}
}
sqlx::query(
"CREATE TRIGGER reject_replay BEFORE INSERT ON thread_turns
WHEN NEW.turn_id = 'turn-1'
BEGIN SELECT RAISE(FAIL, 'replayed committed prefix'); END",
)
.execute(&pool)
.await
.expect("reject replay of committed rows");
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("retry uncommitted suffix");
let turn_count =
sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM thread_turns WHERE thread_id = ?")
.bind(thread_id.to_string())
.fetch_one(&pool)
.await
.expect("count committed turns");
assert_eq!(
(projection_state(&pool, thread_id).await, turn_count),
(
(i64::try_from(valid_len).expect("rollout length"), 2051),
2049
)
);
}
}
#[tokio::test]
async fn paused_projection_releases_writer_and_retains_progress_when_stopped() {
enum Stop {
Disconnect,
Cancel,
}
for stop in [Stop::Disconnect, Stop::Cancel] {
let home = TempDir::new().expect("temp dir");
let store = Arc::new(projection_store(home.path()).await);
let thread_id = ThreadId::default();
let other_thread_id = ThreadId::default();
for id in [thread_id, other_thread_id] {
create_paginated_thread(&store, id).await;
store
.persist_thread(id, PersistContext::Standard)
.await
.expect("persist session metadata");
}
let pool = store.thread_history_db().await.expect("history db");
let before = projection_state(pool, thread_id).await;
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
append_suffix(
rollout_path.as_path(),
&format!("{}\n", rollout_line(Some(1), turn_started("turn-1"))),
);
let (_, next_offset) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 1);
let (sender, receiver) = tokio::sync::mpsc::channel(2);
super::read_projection_batches(
rollout_path.as_path(),
before.0 as u64,
before.1 as u64,
thread_id,
/*subagent_history_start_ordinal*/ None,
|| false,
|projections, next_offset| {
assert!(
sender
.try_send(super::pipeline::ProjectionMessage::Batch {
projections,
next_offset,
})
.is_ok(),
"queue the complete record"
);
Ok(())
},
)
.expect("read batch");
let consumer_store = Arc::clone(&store);
let consumer = tokio::spawn(async move {
super::pipeline::apply_batches(
&consumer_store,
thread_id,
before.0 as u64,
/*initial_ordinal*/ 0,
receiver,
)
.await
});
// Keep the producer connected but idle until its first batch is visibly committed.
while projection_state(pool, thread_id).await == before {
tokio::task::yield_now().await;
}
assert_eq!(projection_state(pool, thread_id).await, (next_offset, 2));
store
.append_items(AppendThreadItemsParams {
thread_id: other_thread_id,
items: vec![turn_started("unrelated-turn")],
})
.await
.expect("append while the reader is idle");
store
.flush_thread(other_thread_id)
.await
.expect("flush unrelated writer");
assert_eq!(projection_state(pool, other_thread_id).await.1, 2);
match stop {
Stop::Disconnect => {
drop(sender);
consumer
.await
.expect("join consumer")
.expect_err("missing completion");
}
Stop::Cancel => {
consumer.abort();
assert!(
consumer
.await
.expect_err("cancelled consumer")
.is_cancelled()
);
sender.closed().await;
}
}
assert_eq!(projection_state(pool, thread_id).await, (next_offset, 2));
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("resume after stopped producer");
assert_eq!(projection_state(pool, thread_id).await, (next_offset, 2));
}
}
#[tokio::test]
async fn catch_up_rejects_missing_rollout_after_projection() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
store
.shutdown_thread(thread_id)
.await
.expect("close rollout");
fs::remove_file(rollout_path.as_path()).expect("remove rollout");
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect_err("missing projected rollout should fail");
}
#[tokio::test]
async fn sqlite_failure_does_not_fail_durable_jsonl_write() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
fs::create_dir(store.config.sqlite.thread_history_db_path())
.expect("block thread history database");
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
.expect("durable JSONL append should succeed");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let (items, _, _) = RolloutRecorder::load_rollout_items(rollout_path.as_path())
.await
.expect("load durable rollout");
assert!(items.iter().any(|item| {
matches!(
item,
RolloutItem::EventMsg(EventMsg::TurnStarted(event))
if event.turn_id == "turn-1"
)
}));
}
#[tokio::test]
async fn blank_and_rejected_rollout_lines_advance_projection() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let before = projection_state(&pool, thread_id).await;
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let mut file = fs::OpenOptions::new()
.append(true)
.open(rollout_path.as_path())
.expect("open rollout for rejected line");
file.write_all(b"{not json}\n\n")
.expect("append blank and rejected lines");
file.flush().expect("flush rejected line");
let rejected_tail_len = i64::try_from(
fs::metadata(rollout_path.as_path())
.expect("rollout metadata")
.len(),
)
.expect("rollout length");
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("skip rejected tail");
assert_eq!(
projection_state(&pool, thread_id).await,
(rejected_tail_len, before.1)
);
let recorder = store
.live_recorders
.lock()
.await
.get(&thread_id)
.expect("live recorder")
.recorder
.clone();
recorder
.record_canonical_items(&[turn_started("turn-1")])
.await
.expect("queue valid retry");
recorder.flush().await.expect("flush valid retry");
// Replay this small file, cancelling between records.
let checks = Cell::new(0);
assert!(
super::read_projection_batches(
rollout_path.as_path(),
/*start_offset*/ 0,
/*expected_ordinal*/ 0,
thread_id,
/*subagent_history_start_ordinal*/ None,
|| checks.replace(checks.get() + 1) >= 6,
|_batch, _next_offset| Ok(()),
)
.is_err()
);
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("project valid retry after rejected line");
let (expected_start_byte_offset, _) =
rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 1);
let start_byte_offset = sqlx::query_scalar::<_, Option<i64>>(
"SELECT rollout_byte_offset FROM thread_turns WHERE thread_id = ? AND turn_id = ?",
)
.bind(thread_id.to_string())
.bind("turn-1")
.fetch_one(&pool)
.await
.expect("read projected turn byte offset");
assert_eq!(start_byte_offset, Some(expected_start_byte_offset));
let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())
.expect("rollout length");
assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 2));
}
#[tokio::test]
async fn unknown_rollout_lines_leave_ordinals_pending_for_valid_retries() {
let unknown_line = |ordinal| {
format!(
concat!(
"{{\"timestamp\":\"2025-01-01T00:00:00.000Z\",\"ordinal\":{ordinal},",
"\"type\":\"future_item\",\"payload\":{{}}}}"
),
ordinal = ordinal
)
};
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let before = projection_state(&pool, thread_id).await;
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
let pending_line = unknown_line(1);
append_suffix(rollout_path.as_path(), format!("{pending_line}\n").as_str());
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("skip unknown tail");
let expected_pending_offset =
before.0 + i64::try_from(pending_line.len() + 1).expect("pending line byte count");
assert_eq!(
projection_state(&pool, thread_id).await,
(expected_pending_offset, 1)
);
append_suffix(
rollout_path.as_path(),
format!(
"{}\n{}\n{}\n",
rollout_line(Some(1), turn_started("retry-turn")),
unknown_line(2),
rollout_line(Some(3), turn_started("turn-1")),
)
.as_str(),
);
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("project valid retry and later history");
let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())
.expect("rollout length");
assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 4));
let turn_ordinals = sqlx::query_as::<_, (String, i64)>(
"SELECT turn_id, rollout_ordinal FROM thread_turns WHERE thread_id = ? ORDER BY rollout_ordinal",
)
.bind(thread_id.to_string())
.fetch_all(&pool)
.await
.expect("read projected turns");
assert_eq!(
turn_ordinals,
vec![("retry-turn".to_string(), 1), ("turn-1".to_string(), 3)]
);
}
#[tokio::test]
async fn event_timestamps_allow_invalid_rollout_timestamps() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let invalid_timestamp_line = |ordinal, item| {
rollout_line(Some(ordinal), item).replace("2025-01-01T00:00:00.000Z", "not-a-timestamp")
};
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
append_suffix(
rollout_path.as_path(),
format!(
"{}\n{}\n",
invalid_timestamp_line(1, turn_started("turn-1")),
invalid_timestamp_line(
2,
completed_item(
thread_id,
"turn-1",
agent_message("agent-1", MessagePhase::FinalAnswer),
),
),
)
.as_str(),
);
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("project records with event timestamps");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let created_at_ms = sqlx::query_scalar::<_, i64>(
"SELECT created_at_ms FROM thread_items WHERE thread_id = ? AND turn_id = ? AND item_id = ?",
)
.bind(thread_id.to_string())
.bind("turn-1")
.bind("agent-1")
.fetch_one(&pool)
.await
.expect("read projected item timestamp");
assert_eq!(created_at_ms, 0);
let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())
.expect("rollout length");
assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 3));
}
#[tokio::test]
async fn malformed_rollout_lines_skip_inferred_ordinal_gaps() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let rollout_path = store
.live_rollout_path(thread_id)
.await
.expect("rollout path");
append_suffix(
rollout_path.as_path(),
format!(
"{{not json}}\n{}\n",
rollout_line(Some(2), turn_started("turn-1"))
)
.as_str(),
);
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
.await
.expect("skip malformed gap and project later history");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())
.expect("rollout length");
assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 3));
let turn_ordinal = sqlx::query_scalar::<_, i64>(
"SELECT rollout_ordinal FROM thread_turns WHERE thread_id = ? AND turn_id = ?",
)
.bind(thread_id.to_string())
.bind("turn-1")
.fetch_one(&pool)
.await
.expect("read projected turn");
assert_eq!(turn_ordinal, 2);
}
#[tokio::test]
async fn shutdown_materializes_items_queued_without_a_flush() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
let recorder = store
.live_recorders
.lock()
.await
.get(&thread_id)
.expect("live recorder")
.recorder
.clone();
recorder
.record_canonical_items(&[turn_started("turn-1")])
.await
.expect("queue rollout item");
store
.shutdown_thread(thread_id)
.await
.expect("shutdown live thread");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let projected_turns = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM thread_turns WHERE thread_id = ? AND turn_id = ?",
)
.bind(thread_id.to_string())
.bind("turn-1")
.fetch_one(&pool)
.await
.expect("read projected turns");
assert_eq!(projected_turns, 1);
}
#[tokio::test]
async fn delete_waits_for_in_flight_projection_before_removing_rows() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
create_paginated_thread(&store, thread_id).await;
store
.persist_thread(thread_id, PersistContext::Standard)
.await
.expect("persist session metadata");
let write_permit = store.live_writer_locks.lock(thread_id).await;
let append_store = store.clone();
let append = tokio::spawn(async move {
append_store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![turn_started("turn-1")],
})
.await
});
tokio::time::sleep(Duration::from_millis(/*millis*/ 10)).await;
let delete_store = store.clone();
let delete = tokio::spawn(async move {
delete_store
.delete_thread(DeleteThreadParams { thread_id })
.await
});
tokio::time::sleep(Duration::from_millis(/*millis*/ 10)).await;
assert!(!delete.is_finished());
drop(write_permit);
append
.await
.expect("join append")
.expect("finish in-flight append");
delete.await.expect("join delete").expect("delete thread");
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
home.path().abs(),
))
.await
.expect("open thread history db");
let counts = sqlx::query_as::<_, (i64, i64, i64)>(
r#"
SELECT
(SELECT COUNT(*) FROM thread_turns WHERE thread_id = ?),
(SELECT COUNT(*) FROM thread_items WHERE thread_id = ?),
(SELECT COUNT(*) FROM thread_history_projection_state WHERE thread_id = ?)
"#,
)
.bind(thread_id.to_string())
.bind(thread_id.to_string())
.bind(thread_id.to_string())
.fetch_one(&pool)
.await
.expect("read history row counts");
assert_eq!(counts, (0, 0, 0));
}
async fn projection_store(codex_home: &Path) -> LocalThreadStore {
let config = test_config(codex_home);
let state_db = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await
.expect("initialize state database for paginated history");
LocalThreadStore::new(config, Some(state_db))
}
async fn create_paginated_thread(store: &LocalThreadStore, thread_id: ThreadId) {
create_paginated_subagent_thread(
store, thread_id, /*history_base*/ None, /*subagent_history_start_ordinal*/ None,
)
.await;
}
async fn prepare_paginated_fork(
store: &LocalThreadStore,
thread_id: ThreadId,
boundary: ForkBoundary,
) -> PreparedFork {
store
.prepare_fork(PrepareForkParams {
thread_id,
boundary,
})
.await
.expect("prepare paginated fork")
}
async fn create_paginated_subagent_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
history_base: Option<HistoryPosition>,
subagent_history_start_ordinal: Option<u64>,
) {
store
.create_thread(CreateThreadParams {
attached_thread: false,
creator_user_id: None,
creator_account_id: None,
session_id: thread_id.into(),
thread_id,
extra_config: None,
forked_from_id: None,
parent_thread_id: None,
source: SessionSource::Exec,
thread_source: None,
originator: "test_originator".to_string(),
base_instructions: BaseInstructions::default(),
dynamic_tools: Vec::new(),
selected_capability_roots: Vec::new(),
multi_agent_version: None,
history_mode: ThreadHistoryMode::Paginated,
history_base,
subagent_history_start_ordinal,
initial_window_id: "window-1".to_string(),
runtime_workspace_roots: None,
metadata: ThreadPersistenceMetadata {
cwd: Some(std::env::current_dir().expect("cwd")),
model_provider: "test-provider".to_string(),
memory_mode: ThreadMemoryMode::Enabled,
},
})
.await
.expect("create paginated thread");
}
fn turn_started(turn_id: &str) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
turn_attribution: None,
turn_id: turn_id.to_string(),
root_turn_id: None,
trace_id: None,
started_at: Some(10),
model_context_window: None,
collaboration_mode_kind: Default::default(),
}))
}
fn turn_started_with_root(turn_id: &str, root_turn_id: &str) -> RolloutItem {
let mut item = turn_started(turn_id);
if let RolloutItem::EventMsg(EventMsg::TurnStarted(event)) = &mut item {
event.root_turn_id = Some(root_turn_id.to_string());
}
item
}
fn turn_completed(turn_id: &str) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
root_turn_id: None,
turn_id: turn_id.to_string(),
last_agent_message: None,
error: None,
started_at: Some(10),
completed_at: Some(20),
duration_ms: Some(10_000),
time_to_first_token_ms: None,
}))
}
fn user_message(message: &str) -> RolloutItem {
RolloutItem::ResponseItem(
ResponseItem::Message {
status: None,
encrypted_content: None,
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputText {
text: message.to_string(),
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
}
.into(),
)
}
fn contains_user_message(items: &[RolloutItem], expected: &str) -> bool {
items.iter().any(|item| {
let RolloutItem::ResponseItem(response_item) = item else {
return false;
};
let ResponseItem::Message { content, .. } = &response_item.item else {
return false;
};
content
.iter()
.any(|content| matches!(content, ContentItem::InputText { text } if text == expected))
})
}
fn completed_item(thread_id: ThreadId, turn_id: &str, item: TurnItem) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
thread_id,
turn_id: turn_id.to_string(),
item,
started_at_ms: Some(0),
completed_at_ms: 1,
}))
}
fn agent_message(id: &str, phase: MessagePhase) -> TurnItem {
TurnItem::AgentMessage(AgentMessageItem {
id: id.to_string(),
content: vec![AgentMessageContent::Text {
text: id.to_string(),
}],
phase: Some(phase),
memory_citation: None,
delivery: None,
questions: None,
})
}
fn rollout_line_byte_offsets(path: &std::path::Path, ordinal: u64) -> (i64, i64) {
let bytes = fs::read(path).expect("read rollout");
let mut start_byte_offset = 0;
for line in bytes.split_inclusive(|byte| *byte == b'\n') {
let end_byte_offset = start_byte_offset + line.len();
if codex_rollout::parse_rollout_line_bytes(line)
.ok()
.and_then(|line| line.ordinal)
== Some(ordinal)
{
return (
i64::try_from(start_byte_offset).expect("start byte offset fits i64"),
i64::try_from(end_byte_offset).expect("end byte offset fits i64"),
);
}
start_byte_offset = end_byte_offset;
}
panic!("missing rollout ordinal {ordinal}");
}
fn compress_rollout(path: &Path) {
let mut compressed_path = path.as_os_str().to_os_string();
compressed_path.push(".zst");
let compressed = zstd::stream::encode_all(
fs::File::open(path).expect("open rollout for compression"),
/*level*/ 0,
)
.expect("compress rollout");
fs::write(compressed_path, compressed).expect("write compressed rollout");
fs::remove_file(path).expect("remove plain rollout");
}
async fn projection_state(pool: &sqlx::SqlitePool, thread_id: ThreadId) -> (i64, i64) {
sqlx::query_as::<_, (i64, i64)>(
r#"
SELECT next_rollout_byte_offset, next_rollout_ordinal
FROM thread_history_projection_state
WHERE thread_id = ?
"#,
)
.bind(thread_id.to_string())
.fetch_one(pool)
.await
.expect("read projection state")
}
fn rollout_line(ordinal: Option<u64>, item: RolloutItem) -> String {
serde_json::to_string(&RolloutLine {
timestamp: "2025-01-01T00:00:00.000Z".to_string(),
ordinal,
item,
})
.expect("serialize rollout line")
}
fn append_suffix(rollout_path: &std::path::Path, suffix: &str) {
let mut file = fs::OpenOptions::new()
.append(true)
.open(rollout_path)
.expect("open rollout suffix");
file.write_all(suffix.as_bytes()).expect("append suffix");
file.flush().expect("flush suffix");
}