Lightweight coding agent that runs in your terminal
Rust
Move v2 extraction chunking into the memory writer (#43808)
jife7f5de0a6ac9
diff --git a/codex-rs/core/src/context/memory.rs b/codex-rs/core/src/context/memory.rsindex 3b1aefc1d6..d96afebd53 100644--- a/codex-rs/core/src/context/memory.rs+++ b/codex-rs/core/src/context/memory.rs@@ -1,8 +1,7 @@-//! Typed, bounded v2 memory context. Extraction chunks preserve selected evidence in order.+//! Typed, bounded v2 memory context.use super::ContextualUserFragment;use codex_protocol::models::ContentItemKind;-use codex_protocol::models::ResponseItem;use codex_utils_output_truncation::TruncationPolicy;use codex_utils_output_truncation::truncate_text;@@ -12,21 +11,6 @@ pub enum MemoryContextFragment {ExtractionEvidence(String),}-impl MemoryContextFragment {- /// Splits already-budgeted extraction input without dropping evidence.- pub fn extraction_messages(mut text: &str) -> Vec<ResponseItem> {- let mut messages = Vec::new();- while !text.is_empty() {- let end = text.floor_char_boundary(text.len().min(8_900));- messages.push(ContextualUserFragment::into(Self::ExtractionEvidence(- text[..end].to_string(),- )));- text = &text[end..];- }- messages- }-}-impl ContextualUserFragment for MemoryContextFragment {fn role(&self) -> &'static str {match self {@@ -57,7 +41,3 @@ impl ContextualUserFragment for MemoryContextFragment {truncate_text(text, TruncationPolicy::Bytes(8_900))}}--#[cfg(test)]-#[path = "memory_tests.rs"]-mod tests;diff --git a/codex-rs/core/src/context/memory_tests.rs b/codex-rs/core/src/context/memory_tests.rsdeleted file mode 100644index b8b687533b..0000000000--- a/codex-rs/core/src/context/memory_tests.rs+++ /dev/null@@ -1,21 +0,0 @@-use super::*;-use codex_protocol::models::ContentItem;-use pretty_assertions::assert_eq;--#[test]-fn extraction_chunks_preserve_unicode_evidence_with_bounded_messages() {- let evidence = "User correction: π\n".repeat(2_000);- let mut reconstructed = String::new();- for message in MemoryContextFragment::extraction_messages(&evidence) {- let ResponseItem::Message { role, content, .. } = message else {- panic!("message")- };- assert_eq!(role, "user");- let [ContentItem::InputText { text }] = content.as_slice() else {- panic!("text")- };- assert!(text.len() < 9_000);- reconstructed.push_str(text);- }- assert_eq!(reconstructed, evidence);-}diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rsindex ba6472b0fb..1fa128c2f5 100644--- a/codex-rs/memories/write/src/phase1.rs+++ b/codex-rs/memories/write/src/phase1.rs@@ -297,9 +297,7 @@ mod job {phase: None,internal_chat_message_metadata_passthrough: None,}],- MemoryVersion::V2 => {- codex_core::context::MemoryContextFragment::extraction_messages(&input_text)- }+ MemoryVersion::V2 => crate::rollout_input::extraction_messages(&input_text),};prompt.base_instructions = BaseInstructions {text: match config.memories.version {diff --git a/codex-rs/memories/write/src/rollout_input.rs b/codex-rs/memories/write/src/rollout_input.rsindex 1e99325cfd..7a1d1d94c3 100644--- a/codex-rs/memories/write/src/rollout_input.rs+++ b/codex-rs/memories/write/src/rollout_input.rs@@ -1,7 +1,10 @@//! Selects v2 extraction evidence by provenance, then restores chronological order.//! Human replies retain their questions; message media is replaced with placeholders.//! Budgets use the same token estimate as the rest of the memory pipeline.+//! Extraction chunks preserve selected evidence in order within bounded messages.+use codex_core::context::ContextualUserFragment;+use codex_core::context::MemoryContextFragment;use codex_protocol::ToolName;use codex_protocol::models::ContentItem;use codex_protocol::models::MessagePhase;@@ -241,6 +244,19 @@ pub(crate) fn serialize_tiered_input(Ok(rendered)}+/// Splits already-budgeted extraction input without dropping evidence.+pub(crate) fn extraction_messages(mut text: &str) -> Vec<ResponseItem> {+ let mut messages = Vec::new();+ while !text.is_empty() {+ let end = text.floor_char_boundary(text.len().min(8_900));+ messages.push(ContextualUserFragment::into(+ MemoryContextFragment::ExtractionEvidence(text[..end].to_string()),+ ));+ text = &text[end..];+ }+ messages+}+pub(crate) fn sanitize_response_item_for_memories(item: &ResponseItem) -> Option<ResponseItem> {let ResponseItem::Message {id,diff --git a/codex-rs/memories/write/src/rollout_input_tests.rs b/codex-rs/memories/write/src/rollout_input_tests.rsindex c2f4baafd8..5cdd2baeb5 100644--- a/codex-rs/memories/write/src/rollout_input_tests.rs+++ b/codex-rs/memories/write/src/rollout_input_tests.rs@@ -1,6 +1,24 @@use super::*;use pretty_assertions::assert_eq;+#[test]+fn extraction_chunks_preserve_unicode_evidence_with_bounded_messages() {+ let evidence = "User correction: π\n".repeat(2_000);+ let mut reconstructed = String::new();+ for message in extraction_messages(&evidence) {+ let ResponseItem::Message { role, content, .. } = message else {+ panic!("message")+ };+ assert_eq!(role, "user");+ let [ContentItem::InputText { text }] = content.as_slice() else {+ panic!("text")+ };+ assert!(text.len() < 9_000);+ reconstructed.push_str(text);+ }+ assert_eq!(reconstructed, evidence);+}+#[test]fn classifies_memory_excluded_fragments() {let cases = [