Lightweight coding agent that runs in your terminal
Rust
Make thread history projection resilient to malformed rollouts (#36188)
Owen Lin6256a7ccc794
diff --git a/codex-rs/thread-store/src/local/thread_history.rs b/codex-rs/thread-store/src/local/thread_history.rsindex b20cc17989..c89d1b28b2 100644--- a/codex-rs/thread-store/src/local/thread_history.rs+++ b/codex-rs/thread-store/src/local/thread_history.rs@@ -21,16 +21,27 @@ pub(super) use turn_lookup::find_visible_turn;/// A valid complete rollout line with its absolute byte span in durable JSONL.///-/// `start_byte_offset..end_byte_offset` includes the terminating newline. Blank and rejected-/// lines do not produce a value here, but still advance later spans.+/// `start_byte_offset..end_byte_offset` includes the terminating newline.pub(super) struct ProjectedRolloutLine {pub ordinal: u64,pub start_byte_offset: u64,pub end_byte_offset: u64,- pub created_at_ms: i64,+ pub fallback_created_at_ms: Option<i64>,pub changes: ThreadHistoryChangeSet,}+/// One ordered update to apply while advancing a rollout projection checkpoint.+///+/// Skipped ordinal ranges keep the byte and ordinal checkpoints describing the same durable+/// prefix even when a complete rollout line cannot be projected.+pub(super) enum RolloutProjectionStep {+ Line(ProjectedRolloutLine),+ SkippedOrdinalRange {+ start_ordinal: u64,+ end_ordinal_exclusive: u64,+ },+}+pub(super) struct RolloutProjectionState {pub next_byte_offset: u64,pub next_ordinal: u64,@@ -91,7 +102,7 @@ pub(super) async fn apply_projection(start_offset: u64,next_offset: u64,initial_ordinal: u64,- projections: Vec<ProjectedRolloutLine>,+ projections: Vec<RolloutProjectionStep>,) -> ThreadStoreResult<()> {let pool = store.thread_history_db().await?;// Write the projected rows and advance the JSONL offset and ordinal in one transaction. If@@ -123,29 +134,57 @@ WHERE thread_id = ?}for projection in projections {- let ordinal = sqlite_integer(projection.ordinal, "rollout ordinal")?;- if ordinal != next_ordinal {- return Err(ThreadStoreError::Internal {- message: format!(- "thread history projection for {thread_id} expected ordinal {next_ordinal}, got {ordinal}"- ),- });+ match projection {+ RolloutProjectionStep::Line(projection) => {+ let ordinal = sqlite_integer(projection.ordinal, "rollout ordinal")?;+ if ordinal != next_ordinal {+ return Err(ThreadStoreError::Internal {+ message: format!(+ "thread history projection for {thread_id} expected ordinal {next_ordinal}, got {ordinal}"+ ),+ });+ }+ apply_change_set(+ &mut transaction,+ thread_id.as_str(),+ ordinal,+ sqlite_integer(projection.start_byte_offset, "rollout byte offset")?,+ sqlite_integer(projection.end_byte_offset, "rollout byte offset")?,+ projection.fallback_created_at_ms,+ projection.changes,+ )+ .await?;+ next_ordinal =+ next_ordinal+ .checked_add(1)+ .ok_or_else(|| ThreadStoreError::Internal {+ message: "rollout ordinal exceeds SQLite integer range".to_string(),+ })?;+ }+ RolloutProjectionStep::SkippedOrdinalRange {+ start_ordinal,+ end_ordinal_exclusive,+ } => {+ let start_ordinal = sqlite_integer(start_ordinal, "rollout ordinal")?;+ if start_ordinal != next_ordinal {+ return Err(ThreadStoreError::Internal {+ message: format!(+ "thread history projection for {thread_id} expected ordinal {next_ordinal}, got {start_ordinal}"+ ),+ });+ }+ let end_ordinal_exclusive =+ sqlite_integer(end_ordinal_exclusive, "rollout ordinal")?;+ if end_ordinal_exclusive <= start_ordinal {+ return Err(ThreadStoreError::Internal {+ message: format!(+ "thread history projection for {thread_id} has an empty skipped ordinal range"+ ),+ });+ }+ next_ordinal = end_ordinal_exclusive;+ }}- apply_change_set(- &mut transaction,- thread_id.as_str(),- ordinal,- sqlite_integer(projection.start_byte_offset, "rollout byte offset")?,- sqlite_integer(projection.end_byte_offset, "rollout byte offset")?,- projection.created_at_ms,- projection.changes,- )- .await?;- next_ordinal = next_ordinal- .checked_add(1)- .ok_or_else(|| ThreadStoreError::Internal {- message: "rollout ordinal exceeds SQLite integer range".to_string(),- })?;}sqlx::query(@@ -214,7 +253,7 @@ async fn apply_change_set(rollout_ordinal: i64,rollout_byte_offset: i64,rollout_end_byte_offset: i64,- created_at_ms: i64,+ fallback_created_at_ms: Option<i64>,changes: ThreadHistoryChangeSet,) -> ThreadStoreResult<()> {for turn in changes.changed_turns {@@ -342,6 +381,14 @@ WHERE thread_id = ?}for item in changes.changed_items {+ let created_at_ms =+ item.started_at_ms+ .or(fallback_created_at_ms)+ .ok_or_else(|| ThreadStoreError::Internal {+ message: format!(+ "thread history projection for {thread_id} is missing an item creation timestamp"+ ),+ })?;let item_id = item.item.id().to_string();let item_json = serde_json::to_string(&item.item).map_err(thread_history_error)?;// Completed items are immutable: local producers emit ItemCompleted exactly once perdiff --git a/codex-rs/thread-store/src/local/thread_history_materialization.rs b/codex-rs/thread-store/src/local/thread_history_materialization.rsindex 496f463f7c..9dced72551 100644--- a/codex-rs/thread-store/src/local/thread_history_materialization.rs+++ b/codex-rs/thread-store/src/local/thread_history_materialization.rs@@ -12,15 +12,10 @@ use tracing::warn;use super::LocalThreadStore;use super::thread_history::ProjectedRolloutLine;+use super::thread_history::RolloutProjectionStep;use crate::ThreadStoreError;use crate::ThreadStoreResult;-struct CompleteRolloutLine {- line: RolloutLine,- start_byte_offset: u64,- end_byte_offset: u64,-}-pub(super) async fn materialize_to_sqlite(store: &LocalThreadStore,thread_id: ThreadId,@@ -29,12 +24,15 @@ pub(super) async fn materialize_to_sqlite(if store.state_db.is_none() {return Ok(());}- let start_offset = super::thread_history::projection_state(store, thread_id)- .await?+ let projection_state = super::thread_history::projection_state(store, thread_id).await?;+ let start_offset = projection_state+ .as_ref().map_or(0, |state| state.next_byte_offset);- let (lines, next_offset) = read_complete_rollout_lines(rollout_path, start_offset).await?;- // Empty valid records can still consume bytes through blank or rejected complete lines.- if lines.is_empty() && start_offset == next_offset {+ if projection_state.is_none()+ && !tokio::fs::try_exists(rollout_path)+ .await+ .map_err(thread_store_io_error)?+ {return Ok(());}let session_meta = codex_rollout::read_session_meta_line(rollout_path)@@ -45,31 +43,21 @@ pub(super) async fn materialize_to_sqlite(.history_base.map_or(0, |base| base.end_ordinal_exclusive);let subagent_history_start_ordinal = session_meta.subagent_history_start_ordinal;-- let projections = lines- .iter()- .map(|record| {- let line = &record.line;- let ordinal = line.ordinal.ok_or_else(|| ThreadStoreError::Internal {- message: format!("paginated rollout line for {thread_id} is missing an ordinal"),- })?;- let created_at_ms = DateTime::parse_from_rfc3339(line.timestamp.as_str())- .map(|timestamp| timestamp.timestamp_millis())- .map_err(thread_history_error)?;- let changes = if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) {- ThreadHistoryChangeSet::default()- } else {- project_rollout_line(line)- };- Ok(ProjectedRolloutLine {- ordinal,- start_byte_offset: record.start_byte_offset,- end_byte_offset: record.end_byte_offset,- created_at_ms,- changes,- })- })- .collect::<ThreadStoreResult<Vec<_>>>()?;+ let expected_ordinal = projection_state+ .as_ref()+ .map_or(initial_ordinal, |state| state.next_ordinal);+ let (projections, next_offset) = read_projection_steps(+ rollout_path,+ start_offset,+ expected_ordinal,+ thread_id,+ subagent_history_start_ordinal,+ )+ .await?;+ // Empty valid records can still consume bytes through blank complete lines.+ if projections.is_empty() && start_offset == next_offset {+ return Ok(());+ }super::thread_history::apply_projection(store,thread_id,@@ -81,11 +69,14 @@ pub(super) async fn materialize_to_sqlite(.await}-async fn read_complete_rollout_lines(+async fn read_projection_steps(rollout_path: &Path,start_offset: u64,-) -> ThreadStoreResult<(Vec<CompleteRolloutLine>, u64)> {- let next_offset = match tokio::fs::metadata(rollout_path).await {+ expected_ordinal: u64,+ thread_id: ThreadId,+ subagent_history_start_ordinal: Option<u64>,+) -> ThreadStoreResult<(Vec<RolloutProjectionStep>, u64)> {+ let file_end_offset = match tokio::fs::metadata(rollout_path).await {Ok(metadata) => metadata.len(),Err(err) if err.kind() == std::io::ErrorKind::NotFound && start_offset == 0 => {return Ok((Vec::new(), 0));@@ -93,7 +84,7 @@ async fn read_complete_rollout_lines(Err(err) => return Err(thread_store_io_error(err)),};let byte_count =- next_offset+ file_end_offset.checked_sub(start_offset).ok_or_else(|| ThreadStoreError::Internal {message: "durable rollout shrank before projection".to_string(),@@ -117,19 +108,13 @@ async fn read_complete_rollout_lines(.iter().rposition(|byte| *byte == b'\n').map_or(0, |index| index + 1);- let next_offset = start_offset- .checked_add(u64::try_from(complete_byte_count).map_err(|_| {- ThreadStoreError::Internal {- message: "durable rollout append exceeds addressable memory".to_string(),- }- })?)- .ok_or_else(|| ThreadStoreError::Internal {- message: "durable rollout byte offset overflow".to_string(),- })?;- let mut lines = Vec::new();+ let mut projections = Vec::new();+ let mut next_ordinal = expected_ordinal;+ let mut next_offset = start_offset;+ let mut pending_rejected_line_count = 0;let mut line_start_offset = start_offset;- // Preserve each complete physical line's trailing newline so byte offsets advance through- // every durable byte, including blank or rejected lines that do not project a row.+ // Keep rejected lines pending until a later valid ordinal proves whether they consumed history.+ // This lets a same-ordinal retry replace a failed write without advancing only one checkpoint.for line_bytes in bytes[..complete_byte_count].split_inclusive(|byte| *byte == b'\n') {let line_end_offset = line_start_offset.checked_add(u64::try_from(line_bytes.len()).map_err(|_| {@@ -140,35 +125,153 @@ async fn read_complete_rollout_lines(.ok_or_else(|| ThreadStoreError::Internal {message: "durable rollout byte offset overflow".to_string(),})?;- // Blank physical lines consume bytes but are not rollout records.- if !line_bytes.iter().all(u8::is_ascii_whitespace) {- match serde_json::from_slice::<serde_json::Value>(line_bytes)- .and_then(serde_json::from_value::<RolloutLine>)- {- Ok(line) => lines.push(CompleteRolloutLine {- line,- start_byte_offset: line_start_offset,- end_byte_offset: line_end_offset,- }),+ if line_bytes.iter().all(u8::is_ascii_whitespace) {+ if pending_rejected_line_count == 0 {+ next_offset = line_end_offset;+ }+ line_start_offset = line_end_offset;+ continue;+ }+ let value = match serde_json::from_slice::<serde_json::Value>(line_bytes) {+ Ok(value) => value,+ Err(err) => {+ warn!(+ thread_id = %thread_id,+ rollout_path = %rollout_path.display(),+ line_start_byte_offset = line_start_offset,+ line_end_byte_offset = line_end_offset,+ expected_ordinal = next_ordinal,+ error = %err,+ "deferring rejected rollout line until a later ordinal resolves it"+ );+ pending_rejected_line_count += 1;+ line_start_offset = line_end_offset;+ continue;+ }+ };+ let value_ordinal = value.get("ordinal").and_then(serde_json::Value::as_u64);+ let line = match serde_json::from_value::<RolloutLine>(value) {+ Ok(line) => Some(line),+ Err(err) => {+ warn!(+ thread_id = %thread_id,+ rollout_path = %rollout_path.display(),+ line_start_byte_offset = line_start_offset,+ line_end_byte_offset = line_end_offset,+ expected_ordinal = next_ordinal,+ line_ordinal = ?value_ordinal,+ error = %err,+ "deferring unknown rollout line until a later ordinal resolves it"+ );+ None+ }+ };+ let ordinal = match line+ .as_ref()+ .and_then(|line| line.ordinal)+ .or(value_ordinal)+ {+ Some(ordinal) => ordinal,+ None if line.is_none() => {+ pending_rejected_line_count += 1;+ line_start_offset = line_end_offset;+ continue;+ }+ None => {+ return Err(ThreadStoreError::Internal {+ message: format!(+ "paginated rollout line for {thread_id} is missing an ordinal"+ ),+ });+ }+ };+ if ordinal < next_ordinal {+ return Err(ThreadStoreError::Internal {+ message: format!(+ "thread history projection for {thread_id} expected ordinal {next_ordinal}, got {ordinal}"+ ),+ });+ }+ let Some(line) = line else {+ pending_rejected_line_count += 1;+ line_start_offset = line_end_offset;+ continue;+ };+ let skipped_ordinal_count = ordinal - next_ordinal;+ if skipped_ordinal_count > pending_rejected_line_count {+ return Err(ThreadStoreError::Internal {+ message: format!(+ "thread history projection for {thread_id} expected ordinal {next_ordinal}, got {ordinal}; {pending_rejected_line_count} rejected rollout lines cannot cover that gap"+ ),+ });+ }+ let changes = if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) {+ ThreadHistoryChangeSet::default()+ } else {+ project_rollout_line(&line)+ };+ let fallback_created_at_ms = if changes+ .changed_items+ .iter()+ .any(|item| item.started_at_ms.is_none())+ {+ match DateTime::parse_from_rfc3339(line.timestamp.as_str()) {+ Ok(timestamp) => Some(timestamp.timestamp_millis()),Err(err) => {- // A failed append can leave a partial record behind. The rollout writer- // repairs its newline before retrying, so skip rejected lines just like the- // canonical rollout loader and keep projecting the valid retry that follows.warn!(- "skipping rejected rollout line while projecting {rollout_path:?}: {err}"+ thread_id = %thread_id,+ rollout_path = %rollout_path.display(),+ line_start_byte_offset = line_start_offset,+ line_end_byte_offset = line_end_offset,+ expected_ordinal = next_ordinal,+ line_ordinal = ordinal,+ error = %err,+ "deferring rollout line with invalid timestamp until a later ordinal resolves it");+ pending_rejected_line_count += 1;+ line_start_offset = line_end_offset;+ continue;}}+ } else {+ None+ };+ if skipped_ordinal_count > 0 {+ warn!(+ thread_id = %thread_id,+ rollout_path = %rollout_path.display(),+ line_start_byte_offset = line_start_offset,+ line_end_byte_offset = line_end_offset,+ expected_ordinal = next_ordinal,+ line_ordinal = ordinal,+ skipped_ordinal_start = next_ordinal,+ skipped_ordinal_end_exclusive = ordinal,+ "skipping rollout ordinal range after rejected lines"+ );+ projections.push(RolloutProjectionStep::SkippedOrdinalRange {+ start_ordinal: next_ordinal,+ end_ordinal_exclusive: ordinal,+ });}+ pending_rejected_line_count = 0;+ let next_line_ordinal =+ ordinal+ .checked_add(1)+ .ok_or_else(|| ThreadStoreError::Internal {+ message: "rollout ordinal exceeds SQLite integer range".to_string(),+ })?;+ projections.push(RolloutProjectionStep::Line(ProjectedRolloutLine {+ ordinal,+ start_byte_offset: line_start_offset,+ end_byte_offset: line_end_offset,+ fallback_created_at_ms,+ changes,+ }));+ next_ordinal = next_line_ordinal;+ next_offset = line_end_offset;line_start_offset = line_end_offset;}- Ok((lines, next_offset))-}--fn thread_history_error(err: impl std::fmt::Display) -> ThreadStoreError {- ThreadStoreError::Internal {- message: format!("failed to project thread history: {err}"),- }+ Ok((projections, next_offset))}fn thread_store_io_error(err: std::io::Error) -> ThreadStoreError {diff --git a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rsindex bd58bf0f27..465d401a74 100644--- a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs+++ b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs@@ -1440,7 +1440,6 @@ async fn paginated_projection_accepts_float_rate_limits_and_later_final_answers(.expect("queue unprojected history");recorder.flush().await.expect("flush unprojected history");assert_eq!(projection_state(&pool, thread_id).await, checkpoint);-super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path()).await.expect("project floating-point rate limits");@@ -1712,6 +1711,13 @@ async fn catch_up_rejects_invalid_complete_suffixes_without_advancing_state() {"out of order ordinal",format!("{}\n", rollout_line(Some(2), turn_started("turn-1"))),),+ (+ "gap larger than rejected prefix",+ format!(+ "{{not json}}\n{}\n",+ rollout_line(Some(3), turn_started("turn-1")),+ ),+ ),];for (name, suffix) in cases {let home = TempDir::new().expect("temp dir");@@ -1852,6 +1858,12 @@ async fn blank_and_rejected_rollout_lines_do_not_poison_projection() {.await.expect("persist session metadata");+ let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(+ home.path().abs(),+ ))+ .await+ .expect("open thread history db");+ let before = projection_state(&pool, thread_id).await;let rollout_path = store.live_rollout_path(thread_id).await@@ -1863,6 +1875,14 @@ async fn blank_and_rejected_rollout_lines_do_not_poison_projection() {file.write_all(b"\n \t\r\n{not json}\n\xff\n").expect("append blank and rejected lines");file.flush().expect("flush rejected line");+ super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())+ .await+ .expect("leave rejected tail pending");+ assert_eq!(+ projection_state(&pool, thread_id).await,+ (before.0 + 5, before.1)+ );+let recorder = store.live_recorders.lock()@@ -1881,11 +1901,6 @@ async fn blank_and_rejected_rollout_lines_do_not_poison_projection() {.await.expect("project valid retry after rejected line");- let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(- home.path().abs(),- ))- .await- .expect("open thread history db");let (expected_start_byte_offset, _) =rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 1);let start_byte_offset = sqlx::query_scalar::<_, Option<i64>>(@@ -1897,6 +1912,198 @@ async fn blank_and_rejected_rollout_lines_do_not_poison_projection() {.await.expect("read projected turn byte offset");assert_eq!(start_byte_offset, Some(expected_start_byte_offset));+ let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())+ .expect("rollout length");+ assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 2));+}++#[tokio::test]+async fn unprojectable_rollout_lines_wait_for_later_ordinals() {+ let unknown_line = |ordinal| {+ format!(+ concat!(+ "{{\"timestamp\":\"2025-01-01T00:00:00.000Z\",\"ordinal\":{ordinal},",+ "\"type\":\"future_item\",\"payload\":{{}}}}"+ ),+ ordinal = ordinal+ )+ };+ let cases = [+ ("unknown payload", unknown_line(1), unknown_line(2)),+ (+ "structurally invalid JSON",+ "{}".to_string(),+ "null".to_string(),+ ),+ ];+ for (name, pending_line, skipped_line) in cases {+ let home = TempDir::new().expect("temp dir");+ let store = projection_store(home.path()).await;+ let thread_id = ThreadId::default();+ create_paginated_thread(&store, thread_id).await;+ store+ .persist_thread(thread_id)+ .await+ .expect("persist session metadata");++ let pool = codex_state::open_thread_history_db(+ &codex_state::SqliteConfig::new_for_testing(home.path().abs()),+ )+ .await+ .expect("open thread history db");+ let before = projection_state(&pool, thread_id).await;+ let rollout_path = store+ .live_rollout_path(thread_id)+ .await+ .expect("rollout path");+ append_suffix(rollout_path.as_path(), format!("{pending_line}\n").as_str());++ super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())+ .await+ .expect("leave unprojectable tail pending");+ assert_eq!(projection_state(&pool, thread_id).await, before, "{name}");++ append_suffix(+ rollout_path.as_path(),+ format!(+ "{}\n{skipped_line}\n{}\n",+ rollout_line(Some(1), turn_started("retry-turn")),+ rollout_line(Some(3), turn_started("turn-1")),+ )+ .as_str(),+ );++ super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())+ .await+ .expect(name);++ let rollout_len =+ i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())+ .expect("rollout length");+ assert_eq!(+ projection_state(&pool, thread_id).await,+ (rollout_len, 4),+ "{name}"+ );+ let turn_ordinals = sqlx::query_as::<_, (String, i64)>(+ "SELECT turn_id, rollout_ordinal FROM thread_turns WHERE thread_id = ? ORDER BY rollout_ordinal",+ )+ .bind(thread_id.to_string())+ .fetch_all(&pool)+ .await+ .expect("read projected turns");+ assert_eq!(+ turn_ordinals,+ vec![("retry-turn".to_string(), 1), ("turn-1".to_string(), 3)],+ "{name}"+ );+ }+}++#[tokio::test]+async fn event_timestamps_allow_invalid_rollout_timestamps() {+ let home = TempDir::new().expect("temp dir");+ let store = projection_store(home.path()).await;+ let thread_id = ThreadId::default();+ create_paginated_thread(&store, thread_id).await;+ store+ .persist_thread(thread_id)+ .await+ .expect("persist session metadata");++ let invalid_timestamp_line = |ordinal, item| {+ rollout_line(Some(ordinal), item).replace("2025-01-01T00:00:00.000Z", "not-a-timestamp")+ };+ let rollout_path = store+ .live_rollout_path(thread_id)+ .await+ .expect("rollout path");+ append_suffix(+ rollout_path.as_path(),+ format!(+ "{}\n{}\n",+ invalid_timestamp_line(1, turn_started("turn-1")),+ invalid_timestamp_line(+ 2,+ completed_item(+ thread_id,+ "turn-1",+ agent_message("agent-1", MessagePhase::FinalAnswer),+ ),+ ),+ )+ .as_str(),+ );++ super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())+ .await+ .expect("project records with event timestamps");++ let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(+ home.path().abs(),+ ))+ .await+ .expect("open thread history db");+ let created_at_ms = sqlx::query_scalar::<_, i64>(+ "SELECT created_at_ms FROM thread_items WHERE thread_id = ? AND turn_id = ? AND item_id = ?",+ )+ .bind(thread_id.to_string())+ .bind("turn-1")+ .bind("agent-1")+ .fetch_one(&pool)+ .await+ .expect("read projected item timestamp");+ assert_eq!(created_at_ms, 0);+ let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())+ .expect("rollout length");+ assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 3));+}++#[tokio::test]+async fn malformed_rollout_lines_skip_inferred_ordinal_gaps() {+ let home = TempDir::new().expect("temp dir");+ let store = projection_store(home.path()).await;+ let thread_id = ThreadId::default();+ create_paginated_thread(&store, thread_id).await;+ store+ .persist_thread(thread_id)+ .await+ .expect("persist session metadata");++ let rollout_path = store+ .live_rollout_path(thread_id)+ .await+ .expect("rollout path");+ append_suffix(+ rollout_path.as_path(),+ format!(+ "{{not json}}\n{}\n",+ rollout_line(Some(2), turn_started("turn-1"))+ )+ .as_str(),+ );++ super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())+ .await+ .expect("skip malformed gap and project later history");++ let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(+ home.path().abs(),+ ))+ .await+ .expect("open thread history db");+ let rollout_len = i64::try_from(fs::metadata(rollout_path).expect("rollout metadata").len())+ .expect("rollout length");+ assert_eq!(projection_state(&pool, thread_id).await, (rollout_len, 3));+ let turn_ordinal = sqlx::query_scalar::<_, i64>(+ "SELECT rollout_ordinal FROM thread_turns WHERE thread_id = ? AND turn_id = ?",+ )+ .bind(thread_id.to_string())+ .bind("turn-1")+ .fetch_one(&pool)+ .await+ .expect("read projected turn");+ assert_eq!(turn_ordinal, 2);}#[tokio::test]