Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Move v2 extraction chunking into the memory writer (#43808)

jife7f5de0a6ac9
diff --git a/codex-rs/core/src/context/memory.rs b/codex-rs/core/src/context/memory.rs
index 3b1aefc1d6..d96afebd53 100644
--- a/codex-rs/core/src/context/memory.rs
+++ b/codex-rs/core/src/context/memory.rs
@@ -1,8 +1,7 @@
-//! Typed, bounded v2 memory context. Extraction chunks preserve selected evidence in order.
+//! Typed, bounded v2 memory context.
use super::ContextualUserFragment;
use codex_protocol::models::ContentItemKind;
-use codex_protocol::models::ResponseItem;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::truncate_text;
@@ -12,21 +11,6 @@ pub enum MemoryContextFragment {
ExtractionEvidence(String),
}
-impl MemoryContextFragment {
- /// Splits already-budgeted extraction input without dropping evidence.
- pub fn extraction_messages(mut text: &str) -> Vec<ResponseItem> {
- let mut messages = Vec::new();
- while !text.is_empty() {
- let end = text.floor_char_boundary(text.len().min(8_900));
- messages.push(ContextualUserFragment::into(Self::ExtractionEvidence(
- text[..end].to_string(),
- )));
- text = &text[end..];
- }
- messages
- }
-}
-
impl ContextualUserFragment for MemoryContextFragment {
fn role(&self) -> &'static str {
match self {
@@ -57,7 +41,3 @@ impl ContextualUserFragment for MemoryContextFragment {
truncate_text(text, TruncationPolicy::Bytes(8_900))
}
}
-
-#[cfg(test)]
-#[path = "memory_tests.rs"]
-mod tests;
diff --git a/codex-rs/core/src/context/memory_tests.rs b/codex-rs/core/src/context/memory_tests.rs
deleted file mode 100644
index b8b687533b..0000000000
--- a/codex-rs/core/src/context/memory_tests.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-use super::*;
-use codex_protocol::models::ContentItem;
-use pretty_assertions::assert_eq;
-
-#[test]
-fn extraction_chunks_preserve_unicode_evidence_with_bounded_messages() {
- let evidence = "User correction: 🐈\n".repeat(2_000);
- let mut reconstructed = String::new();
- for message in MemoryContextFragment::extraction_messages(&evidence) {
- let ResponseItem::Message { role, content, .. } = message else {
- panic!("message")
- };
- assert_eq!(role, "user");
- let [ContentItem::InputText { text }] = content.as_slice() else {
- panic!("text")
- };
- assert!(text.len() < 9_000);
- reconstructed.push_str(text);
- }
- assert_eq!(reconstructed, evidence);
-}
diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rs
index ba6472b0fb..1fa128c2f5 100644
--- a/codex-rs/memories/write/src/phase1.rs
+++ b/codex-rs/memories/write/src/phase1.rs
@@ -297,9 +297,7 @@ mod job {
phase: None,
internal_chat_message_metadata_passthrough: None,
}],
- MemoryVersion::V2 => {
- codex_core::context::MemoryContextFragment::extraction_messages(&input_text)
- }
+ MemoryVersion::V2 => crate::rollout_input::extraction_messages(&input_text),
};
prompt.base_instructions = BaseInstructions {
text: match config.memories.version {
diff --git a/codex-rs/memories/write/src/rollout_input.rs b/codex-rs/memories/write/src/rollout_input.rs
index 1e99325cfd..7a1d1d94c3 100644
--- a/codex-rs/memories/write/src/rollout_input.rs
+++ b/codex-rs/memories/write/src/rollout_input.rs
@@ -1,7 +1,10 @@
//! Selects v2 extraction evidence by provenance, then restores chronological order.
//! Human replies retain their questions; message media is replaced with placeholders.
//! Budgets use the same token estimate as the rest of the memory pipeline.
+//! Extraction chunks preserve selected evidence in order within bounded messages.
+use codex_core::context::ContextualUserFragment;
+use codex_core::context::MemoryContextFragment;
use codex_protocol::ToolName;
use codex_protocol::models::ContentItem;
use codex_protocol::models::MessagePhase;
@@ -241,6 +244,19 @@ pub(crate) fn serialize_tiered_input(
Ok(rendered)
}
+/// Splits already-budgeted extraction input without dropping evidence.
+pub(crate) fn extraction_messages(mut text: &str) -> Vec<ResponseItem> {
+ let mut messages = Vec::new();
+ while !text.is_empty() {
+ let end = text.floor_char_boundary(text.len().min(8_900));
+ messages.push(ContextualUserFragment::into(
+ MemoryContextFragment::ExtractionEvidence(text[..end].to_string()),
+ ));
+ text = &text[end..];
+ }
+ messages
+}
+
pub(crate) fn sanitize_response_item_for_memories(item: &ResponseItem) -> Option<ResponseItem> {
let ResponseItem::Message {
id,
diff --git a/codex-rs/memories/write/src/rollout_input_tests.rs b/codex-rs/memories/write/src/rollout_input_tests.rs
index c2f4baafd8..5cdd2baeb5 100644
--- a/codex-rs/memories/write/src/rollout_input_tests.rs
+++ b/codex-rs/memories/write/src/rollout_input_tests.rs
@@ -1,6 +1,24 @@
use super::*;
use pretty_assertions::assert_eq;
+#[test]
+fn extraction_chunks_preserve_unicode_evidence_with_bounded_messages() {
+ let evidence = "User correction: 🐈\n".repeat(2_000);
+ let mut reconstructed = String::new();
+ for message in extraction_messages(&evidence) {
+ let ResponseItem::Message { role, content, .. } = message else {
+ panic!("message")
+ };
+ assert_eq!(role, "user");
+ let [ContentItem::InputText { text }] = content.as_slice() else {
+ panic!("text")
+ };
+ assert!(text.len() < 9_000);
+ reconstructed.push_str(text);
+ }
+ assert_eq!(reconstructed, evidence);
+}
+
#[test]
fn classifies_memory_excluded_fragments() {
let cases = [