Lightweight coding agent that runs in your terminal
Rust
Bound rollout input for memory extraction (#53940)
jif5670b1c18e86
diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rsindex aa11ce276a..cddb20dadc 100644--- a/codex-rs/memories/write/src/phase1.rs+++ b/codex-rs/memories/write/src/phase1.rs@@ -10,7 +10,6 @@ use crate::runtime::MemoryStartupContext;use crate::runtime::StageOneRequestContext;use codex_config::types::MemoriesConfig;use codex_core::Prompt;-use codex_core::RolloutRecorder;use codex_core::config::Config;use codex_protocol::MemoryVersion;use codex_protocol::ResponseItemId;@@ -264,7 +263,7 @@ mod job {rollout_git_branch: Option<&str>,stage_one_context: &StageOneRequestContext,) -> anyhow::Result<(StageOneOutput, Option<TokenUsage>)> {- let (rollout_items, _, _) = RolloutRecorder::load_rollout_items(rollout_path).await?;+ let rollout_items = crate::rollout_input::read_recent_rollout_items(rollout_path).await?;let rollout_contents = match config.memories.version {MemoryVersion::V1 => serialize_filtered_rollout_response_items(&rollout_items)?,MemoryVersion::V2 => crate::rollout_input::serialize_tiered_input(diff --git a/codex-rs/memories/write/src/rollout_input.rs b/codex-rs/memories/write/src/rollout_input.rsindex 2d45c7b63b..65b4461e85 100644--- a/codex-rs/memories/write/src/rollout_input.rs+++ b/codex-rs/memories/write/src/rollout_input.rs@@ -1,4 +1,4 @@-//! Selects v2 extraction evidence by provenance, then restores chronological order.+//! Reads recent rollout evidence and selects v2 input by provenance in chronological order.//! Human replies retain their questions; message media is replaced with placeholders.//! Budgets use the same token estimate as the rest of the memory pipeline.//! Extraction chunks preserve selected evidence in order within bounded messages.@@ -11,13 +11,80 @@ use codex_protocol::models::MessagePhase;use codex_protocol::models::ResponseItem;use codex_protocol::protocol::InterAgentCommunication;use codex_protocol::request_user_input::RequestUserInputResponse;+use codex_rollout::ReverseJsonlScanner;use codex_rollout::RolloutItem;+use codex_rollout::ScanOutcome;+use codex_rollout::open_rollout_seekable_reader;use codex_rollout::should_persist_response_item_for_memories;use codex_secrets::redact_secrets;use codex_utils_output_truncation::TruncationPolicy;use codex_utils_output_truncation::approx_bytes_for_tokens;use codex_utils_output_truncation::truncate_text;use std::collections::HashMap;+use std::io::BufRead;+use std::io::Read;+use std::path::Path;++const MAX_ROLLOUT_BYTES: usize = 8 * 1024 * 1024;+const MAX_RECORD_BYTES: usize = 1024 * 1024;++/// Keeps the newest sanitized evidence that fits the byte budget, then restores source order.+/// Compressed rollouts use the existing temporary-file reader. Older evidence and records over+/// the per-record limit are intentionally omitted; this is not a complete history for resume.+/// The first line must contain valid session metadata within the same record limit.+pub(crate) async fn read_recent_rollout_items(path: &Path) -> anyhow::Result<Vec<RolloutItem>> {+ let path = path.to_owned();+ let (stop, keep_running) = tokio::sync::oneshot::channel::<()>();+ let task = tokio::task::spawn_blocking(move || -> anyhow::Result<Vec<RolloutItem>> {+ let mut reader = open_rollout_seekable_reader(&path)?;+ // Validate this session's format from its header, not metadata copied into fork history.+ // Bound the header read too: session metadata may contain large instructions or schemas.+ let mut header = String::new();+ std::io::BufReader::new((&mut reader).take(MAX_RECORD_BYTES as u64))+ .read_line(&mut header)?;+ anyhow::ensure!(+ matches!(+ codex_rollout::parse_rollout_line(&header)?.item,+ RolloutItem::SessionMeta(_)+ ),+ "memory extraction requires valid session metadata within the record limit"+ );+ let mut scanner = ReverseJsonlScanner::new(reader)?.with_max_record_bytes(MAX_RECORD_BYTES);+ let mut remaining = MAX_ROLLOUT_BYTES;+ let mut items = Vec::new();+ while !stop.is_closed() {+ let Some(outcome) = scanner.scan_next_rollout_line()? else {+ break;+ };+ let ScanOutcome::Parsed(mut line) = outcome else {+ continue;+ };+ if !matches!(+ &line.item,+ RolloutItem::ResponseItem(_) | RolloutItem::InterAgentCommunication(_)+ ) {+ continue;+ }+ if let RolloutItem::ResponseItem(item) = &mut line.item {+ let Some(sanitized) = sanitize_response_item_for_memories(&item.item) else {+ continue;+ };+ item.item = sanitized;+ }+ let Some(next_remaining) = remaining.checked_sub(serde_json::to_vec(&line.item)?.len())+ else {+ break;+ };+ remaining = next_remaining;+ items.push(line.item);+ }+ items.reverse();+ Ok(items)+ });+ let result = task.await?;+ drop(keep_running);+ result+}const OMITTED: &str = "[... response items omitted ...]\n";const TRUNCATION_RESERVE_BYTES: usize = 96;diff --git a/codex-rs/memories/write/src/rollout_input_tests.rs b/codex-rs/memories/write/src/rollout_input_tests.rsindex 5cdd2baeb5..2878f396f3 100644--- a/codex-rs/memories/write/src/rollout_input_tests.rs+++ b/codex-rs/memories/write/src/rollout_input_tests.rs@@ -1,5 +1,65 @@use super::*;use pretty_assertions::assert_eq;+use std::io::Write;++#[tokio::test]+async fn recent_rollout_items_bound_history_and_preserve_newest_records() -> anyhow::Result<()> {+ let home = tempfile::tempdir()?;+ let path = home.path().join("rollout.jsonl");+ let mut file = std::fs::File::create(&path)?;+ let metadata = serde_json::json!({+ "timestamp": "2026-10-12T00:00:00Z",+ "type": "session_meta",+ "payload": codex_rollout::SessionMeta::default()+ });+ writeln!(file, "{metadata}")?;+ let old = serde_json::json!({+ "timestamp": "2026-10-12T00:00:00Z",+ "type": "response_item",+ "payload": {"type": "message", "role": "user", "content": [+ {"type": "input_text", "text": "older evidence".repeat(/*n*/ 20_000)}+ ]}+ });+ let old = serde_json::to_string(&old)?;+ for _ in 0..(MAX_ROLLOUT_BYTES / old.len() + 2) {+ writeln!(file, "{old}")?;+ }+ let mut newest = Vec::new();+ for text in ["recent question", "latest correction 🦀"] {+ // Neither an oversized nor a malformed record may hide the newest valid evidence.+ writeln!(file, "{}", "x".repeat(MAX_RECORD_BYTES + 1))?;+ file.write_all(b"\xff\n")?;+ let line = serde_json::json!({+ "timestamp": "2026-10-12T00:00:00Z",+ "type": "response_item",+ "payload": {"type": "message", "role": "user", "content": [+ {"type": "input_text", "text": text}+ ]}+ });+ writeln!(file, "{line}")?;+ newest.push(codex_rollout::decode_rollout_line(line)?.item);+ }+ drop(file);++ let items = read_recent_rollout_items(&path).await?;+ assert_eq!(+ serde_json::to_value(&items[items.len() - newest.len()..])?,+ serde_json::to_value(&newest)?+ );+ let retained_bytes = items.iter().try_fold(/*init*/ 0, |bytes, item| {+ serde_json::to_vec(item).map(|json| bytes + json.len())+ })?;+ assert!(retained_bytes <= MAX_ROLLOUT_BYTES);+ let older_item_bytes =+ serde_json::to_vec(&codex_rollout::parse_rollout_line(&old)?.item)?.len();+ assert!(retained_bytes + older_item_bytes > MAX_ROLLOUT_BYTES);++ let mut metadata = metadata;+ metadata["payload"]["history_mode"] = serde_json::json!("future");+ std::fs::write(&path, format!("{metadata}\n{old}\n"))?;+ assert!(read_recent_rollout_items(&path).await.is_err());+ Ok(())+}#[test]fn extraction_chunks_preserve_unicode_evidence_with_bounded_messages() {diff --git a/codex-rs/memories/write/src/startup_tests.rs b/codex-rs/memories/write/src/startup_tests.rsindex 1f2cb94f6b..009b8a6dbe 100644--- a/codex-rs/memories/write/src/startup_tests.rs+++ b/codex-rs/memories/write/src/startup_tests.rs@@ -897,6 +897,20 @@ async fn run_memory_phase_one_model_request_test(contents.push_str(&serde_json::to_string(&line)?);contents.push('\n');}+ // Exercise bounded reading through the actual extraction request, for both memory versions.+ contents.push_str(&"x".repeat(1024 * 1024 + 1));+ contents.push('\n');+ contents.push_str(+ &json!({+ "timestamp": chrono::Utc::now().to_rfc3339(),+ "type": "response_item",+ "payload": {"type": "message", "role": "user", "content": [+ {"type": "input_text", "text": "newest correction after an oversized record"}+ ]}+ })+ .to_string(),+ );+ contents.push('\n');tokio::fs::write(&path, &contents).await?;db.update_thread_git_info(source_id,@@ -935,6 +949,7 @@ async fn run_memory_phase_one_model_request_test(let request = wait_for_single_request(&response).await;let evidence = request.message_input_texts("user").join("");assert!(evidence.contains("replay-bearing assistant evidence"));+ assert!(evidence.contains("newest correction after an oversized record"));assert!(!evidence.contains("encrypted_content"));if version == codex_protocol::MemoryVersion::V1 {assert!(evidence.contains("memory_test_tool"));@@ -1272,7 +1287,18 @@ async fn seed_stage1_candidate(),};let jsonl = serde_json::to_string(&line)?;- tokio::fs::write(&rollout_path, format!("{jsonl}\n")).await?;+ let meta = json!({+ "timestamp": updated_at.to_rfc3339(),+ "type": "session_meta",+ "payload": codex_rollout::SessionMeta {+ id: thread_id,+ session_id: thread_id.into(),+ timestamp: updated_at.to_rfc3339(),+ cwd: codex_home.to_path_buf(),+ ..Default::default()+ }+ });+ tokio::fs::write(&rollout_path, format!("{meta}\n{jsonl}\n")).await?;let mut metadata_builder = codex_state::ThreadMetadataBuilder::new(thread_id,