Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Bound rollout input for memory extraction (#53940)

jif5670b1c18e86
diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rs
index aa11ce276a..cddb20dadc 100644
--- a/codex-rs/memories/write/src/phase1.rs
+++ b/codex-rs/memories/write/src/phase1.rs
@@ -10,7 +10,6 @@ use crate::runtime::MemoryStartupContext;
use crate::runtime::StageOneRequestContext;
use codex_config::types::MemoriesConfig;
use codex_core::Prompt;
-use codex_core::RolloutRecorder;
use codex_core::config::Config;
use codex_protocol::MemoryVersion;
use codex_protocol::ResponseItemId;
@@ -264,7 +263,7 @@ mod job {
rollout_git_branch: Option<&str>,
stage_one_context: &StageOneRequestContext,
) -> anyhow::Result<(StageOneOutput, Option<TokenUsage>)> {
- let (rollout_items, _, _) = RolloutRecorder::load_rollout_items(rollout_path).await?;
+ let rollout_items = crate::rollout_input::read_recent_rollout_items(rollout_path).await?;
let rollout_contents = match config.memories.version {
MemoryVersion::V1 => serialize_filtered_rollout_response_items(&rollout_items)?,
MemoryVersion::V2 => crate::rollout_input::serialize_tiered_input(
diff --git a/codex-rs/memories/write/src/rollout_input.rs b/codex-rs/memories/write/src/rollout_input.rs
index 2d45c7b63b..65b4461e85 100644
--- a/codex-rs/memories/write/src/rollout_input.rs
+++ b/codex-rs/memories/write/src/rollout_input.rs
@@ -1,4 +1,4 @@
-//! Selects v2 extraction evidence by provenance, then restores chronological order.
+//! Reads recent rollout evidence and selects v2 input by provenance in chronological order.
//! Human replies retain their questions; message media is replaced with placeholders.
//! Budgets use the same token estimate as the rest of the memory pipeline.
//! Extraction chunks preserve selected evidence in order within bounded messages.
@@ -11,13 +11,80 @@ use codex_protocol::models::MessagePhase;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::InterAgentCommunication;
use codex_protocol::request_user_input::RequestUserInputResponse;
+use codex_rollout::ReverseJsonlScanner;
use codex_rollout::RolloutItem;
+use codex_rollout::ScanOutcome;
+use codex_rollout::open_rollout_seekable_reader;
use codex_rollout::should_persist_response_item_for_memories;
use codex_secrets::redact_secrets;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::approx_bytes_for_tokens;
use codex_utils_output_truncation::truncate_text;
use std::collections::HashMap;
+use std::io::BufRead;
+use std::io::Read;
+use std::path::Path;
+
+const MAX_ROLLOUT_BYTES: usize = 8 * 1024 * 1024;
+const MAX_RECORD_BYTES: usize = 1024 * 1024;
+
+/// Keeps the newest sanitized evidence that fits the byte budget, then restores source order.
+/// Compressed rollouts use the existing temporary-file reader. Older evidence and records over
+/// the per-record limit are intentionally omitted; this is not a complete history for resume.
+/// The first line must contain valid session metadata within the same record limit.
+pub(crate) async fn read_recent_rollout_items(path: &Path) -> anyhow::Result<Vec<RolloutItem>> {
+ let path = path.to_owned();
+ let (stop, keep_running) = tokio::sync::oneshot::channel::<()>();
+ let task = tokio::task::spawn_blocking(move || -> anyhow::Result<Vec<RolloutItem>> {
+ let mut reader = open_rollout_seekable_reader(&path)?;
+ // Validate this session's format from its header, not metadata copied into fork history.
+ // Bound the header read too: session metadata may contain large instructions or schemas.
+ let mut header = String::new();
+ std::io::BufReader::new((&mut reader).take(MAX_RECORD_BYTES as u64))
+ .read_line(&mut header)?;
+ anyhow::ensure!(
+ matches!(
+ codex_rollout::parse_rollout_line(&header)?.item,
+ RolloutItem::SessionMeta(_)
+ ),
+ "memory extraction requires valid session metadata within the record limit"
+ );
+ let mut scanner = ReverseJsonlScanner::new(reader)?.with_max_record_bytes(MAX_RECORD_BYTES);
+ let mut remaining = MAX_ROLLOUT_BYTES;
+ let mut items = Vec::new();
+ while !stop.is_closed() {
+ let Some(outcome) = scanner.scan_next_rollout_line()? else {
+ break;
+ };
+ let ScanOutcome::Parsed(mut line) = outcome else {
+ continue;
+ };
+ if !matches!(
+ &line.item,
+ RolloutItem::ResponseItem(_) | RolloutItem::InterAgentCommunication(_)
+ ) {
+ continue;
+ }
+ if let RolloutItem::ResponseItem(item) = &mut line.item {
+ let Some(sanitized) = sanitize_response_item_for_memories(&item.item) else {
+ continue;
+ };
+ item.item = sanitized;
+ }
+ let Some(next_remaining) = remaining.checked_sub(serde_json::to_vec(&line.item)?.len())
+ else {
+ break;
+ };
+ remaining = next_remaining;
+ items.push(line.item);
+ }
+ items.reverse();
+ Ok(items)
+ });
+ let result = task.await?;
+ drop(keep_running);
+ result
+}
const OMITTED: &str = "[... response items omitted ...]\n";
const TRUNCATION_RESERVE_BYTES: usize = 96;
diff --git a/codex-rs/memories/write/src/rollout_input_tests.rs b/codex-rs/memories/write/src/rollout_input_tests.rs
index 5cdd2baeb5..2878f396f3 100644
--- a/codex-rs/memories/write/src/rollout_input_tests.rs
+++ b/codex-rs/memories/write/src/rollout_input_tests.rs
@@ -1,5 +1,65 @@
use super::*;
use pretty_assertions::assert_eq;
+use std::io::Write;
+
+#[tokio::test]
+async fn recent_rollout_items_bound_history_and_preserve_newest_records() -> anyhow::Result<()> {
+ let home = tempfile::tempdir()?;
+ let path = home.path().join("rollout.jsonl");
+ let mut file = std::fs::File::create(&path)?;
+ let metadata = serde_json::json!({
+ "timestamp": "2026-10-12T00:00:00Z",
+ "type": "session_meta",
+ "payload": codex_rollout::SessionMeta::default()
+ });
+ writeln!(file, "{metadata}")?;
+ let old = serde_json::json!({
+ "timestamp": "2026-10-12T00:00:00Z",
+ "type": "response_item",
+ "payload": {"type": "message", "role": "user", "content": [
+ {"type": "input_text", "text": "older evidence".repeat(/*n*/ 20_000)}
+ ]}
+ });
+ let old = serde_json::to_string(&old)?;
+ for _ in 0..(MAX_ROLLOUT_BYTES / old.len() + 2) {
+ writeln!(file, "{old}")?;
+ }
+ let mut newest = Vec::new();
+ for text in ["recent question", "latest correction 🦀"] {
+ // Neither an oversized nor a malformed record may hide the newest valid evidence.
+ writeln!(file, "{}", "x".repeat(MAX_RECORD_BYTES + 1))?;
+ file.write_all(b"\xff\n")?;
+ let line = serde_json::json!({
+ "timestamp": "2026-10-12T00:00:00Z",
+ "type": "response_item",
+ "payload": {"type": "message", "role": "user", "content": [
+ {"type": "input_text", "text": text}
+ ]}
+ });
+ writeln!(file, "{line}")?;
+ newest.push(codex_rollout::decode_rollout_line(line)?.item);
+ }
+ drop(file);
+
+ let items = read_recent_rollout_items(&path).await?;
+ assert_eq!(
+ serde_json::to_value(&items[items.len() - newest.len()..])?,
+ serde_json::to_value(&newest)?
+ );
+ let retained_bytes = items.iter().try_fold(/*init*/ 0, |bytes, item| {
+ serde_json::to_vec(item).map(|json| bytes + json.len())
+ })?;
+ assert!(retained_bytes <= MAX_ROLLOUT_BYTES);
+ let older_item_bytes =
+ serde_json::to_vec(&codex_rollout::parse_rollout_line(&old)?.item)?.len();
+ assert!(retained_bytes + older_item_bytes > MAX_ROLLOUT_BYTES);
+
+ let mut metadata = metadata;
+ metadata["payload"]["history_mode"] = serde_json::json!("future");
+ std::fs::write(&path, format!("{metadata}\n{old}\n"))?;
+ assert!(read_recent_rollout_items(&path).await.is_err());
+ Ok(())
+}
#[test]
fn extraction_chunks_preserve_unicode_evidence_with_bounded_messages() {
diff --git a/codex-rs/memories/write/src/startup_tests.rs b/codex-rs/memories/write/src/startup_tests.rs
index 1f2cb94f6b..009b8a6dbe 100644
--- a/codex-rs/memories/write/src/startup_tests.rs
+++ b/codex-rs/memories/write/src/startup_tests.rs
@@ -897,6 +897,20 @@ async fn run_memory_phase_one_model_request_test(
contents.push_str(&serde_json::to_string(&line)?);
contents.push('\n');
}
+ // Exercise bounded reading through the actual extraction request, for both memory versions.
+ contents.push_str(&"x".repeat(1024 * 1024 + 1));
+ contents.push('\n');
+ contents.push_str(
+ &json!({
+ "timestamp": chrono::Utc::now().to_rfc3339(),
+ "type": "response_item",
+ "payload": {"type": "message", "role": "user", "content": [
+ {"type": "input_text", "text": "newest correction after an oversized record"}
+ ]}
+ })
+ .to_string(),
+ );
+ contents.push('\n');
tokio::fs::write(&path, &contents).await?;
db.update_thread_git_info(
source_id,
@@ -935,6 +949,7 @@ async fn run_memory_phase_one_model_request_test(
let request = wait_for_single_request(&response).await;
let evidence = request.message_input_texts("user").join("");
assert!(evidence.contains("replay-bearing assistant evidence"));
+ assert!(evidence.contains("newest correction after an oversized record"));
assert!(!evidence.contains("encrypted_content"));
if version == codex_protocol::MemoryVersion::V1 {
assert!(evidence.contains("memory_test_tool"));
@@ -1272,7 +1287,18 @@ async fn seed_stage1_candidate(
),
};
let jsonl = serde_json::to_string(&line)?;
- tokio::fs::write(&rollout_path, format!("{jsonl}\n")).await?;
+ let meta = json!({
+ "timestamp": updated_at.to_rfc3339(),
+ "type": "session_meta",
+ "payload": codex_rollout::SessionMeta {
+ id: thread_id,
+ session_id: thread_id.into(),
+ timestamp: updated_at.to_rfc3339(),
+ cwd: codex_home.to_path_buf(),
+ ..Default::default()
+ }
+ });
+ tokio::fs::write(&rollout_path, format!("{meta}\n{jsonl}\n")).await?;
let mut metadata_builder = codex_state::ThreadMetadataBuilder::new(
thread_id,