Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add app-server notification media filtering (#41416)

Vivian Fang03147407e3a0
diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs
index d06938d781..7fcfeb8d35 100644
--- a/codex-rs/app-server/src/bespoke_event_handling.rs
+++ b/codex-rs/app-server/src/bespoke_event_handling.rs
@@ -1,5 +1,6 @@
use crate::error_code::internal_error;
use crate::error_code::invalid_request;
+use crate::notification_media::without_notification_media;
use crate::outgoing_message::ClientRequestResult;
use crate::outgoing_message::ThreadScopedOutgoingMessageSender;
use crate::request_processors::populate_thread_turns_from_history;
@@ -91,6 +92,7 @@ use codex_app_server_protocol::guardian_auto_approval_review_notification;
use codex_app_server_protocol::item_event_to_server_notification;
use codex_core::CodexThread;
use codex_core::ThreadManager;
+use codex_features::Feature;
use codex_guardian_v2::StrictReviewReason;
use codex_protocol::ThreadId;
use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool;
@@ -1068,11 +1070,14 @@ pub(crate) async fn apply_bespoke_event_handling(
_ => None,
};
if should_emit {
- let notification = item_event_to_server_notification(
+ let mut notification = item_event_to_server_notification(
EventMsg::ItemStarted(event),
&conversation_id.to_string(),
&event_turn_id,
);
+ if conversation.enabled(Feature::OmitAppServerNotificationMedia) {
+ notification = without_notification_media(notification);
+ }
outgoing.send_server_notification(notification).await;
}
if let Some(params) = dynamic_tool_call_params {
@@ -1093,11 +1098,14 @@ pub(crate) async fn apply_bespoke_event_handling(
&event.item,
)
.await;
- let notification = item_event_to_server_notification(
+ let mut notification = item_event_to_server_notification(
EventMsg::ItemCompleted(event),
&conversation_id.to_string(),
&event_turn_id,
);
+ if conversation.enabled(Feature::OmitAppServerNotificationMedia) {
+ notification = without_notification_media(notification);
+ }
outgoing.send_server_notification(notification).await;
}
msg @ (EventMsg::PatchApplyUpdated(_) | EventMsg::TerminalInteraction(_)) => {
@@ -1129,13 +1137,17 @@ pub(crate) async fn apply_bespoke_event_handling(
.await;
}
EventMsg::RawResponseItem(raw_response_item_event) => {
- maybe_emit_raw_response_item_completed(
- conversation_id,
- &event_turn_id,
- raw_response_item_event.item,
- &outgoing,
- )
- .await;
+ let mut notification = ServerNotification::RawResponseItemCompleted(
+ RawResponseItemCompletedNotification {
+ thread_id: conversation_id.to_string(),
+ turn_id: event_turn_id,
+ item: raw_response_item_event.item,
+ },
+ );
+ if conversation.enabled(Feature::OmitAppServerNotificationMedia) {
+ notification = without_notification_media(notification);
+ }
+ outgoing.send_server_notification(notification).await;
}
EventMsg::RawResponseCompleted(raw_response_completed_event) => {
let notification = RawResponseCompletedNotification {
@@ -1512,22 +1524,6 @@ async fn complete_command_execution_item(
.await;
}
-async fn maybe_emit_raw_response_item_completed(
- conversation_id: ThreadId,
- turn_id: &str,
- item: codex_protocol::models::ResponseItem,
- outgoing: &ThreadScopedOutgoingMessageSender,
-) {
- let notification = RawResponseItemCompletedNotification {
- thread_id: conversation_id.to_string(),
- turn_id: turn_id.to_string(),
- item,
- };
- outgoing
- .send_server_notification(ServerNotification::RawResponseItemCompleted(notification))
- .await;
-}
-
async fn find_and_remove_turn_summary(
_conversation_id: ThreadId,
thread_state: &Arc<Mutex<ThreadState>>,
diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs
index 6aa20e574d..0499145122 100644
--- a/codex-rs/app-server/src/lib.rs
+++ b/codex-rs/app-server/src/lib.rs
@@ -119,6 +119,7 @@ mod mcp_refresh;
mod message_processor;
mod models;
mod models_refresh_worker;
+mod notification_media;
mod otel_reloader;
mod outgoing_message;
mod realtime_event_handling;
diff --git a/codex-rs/app-server/src/notification_media.rs b/codex-rs/app-server/src/notification_media.rs
new file mode 100644
index 0000000000..f6e291bf7f
--- /dev/null
+++ b/codex-rs/app-server/src/notification_media.rs
@@ -0,0 +1,196 @@
+use codex_app_server_protocol::DynamicToolCallOutputContentItem;
+use codex_app_server_protocol::ServerNotification;
+use codex_app_server_protocol::ThreadItem;
+use codex_app_server_protocol::UserInput;
+use codex_protocol::models::ContentItem;
+use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::ResponseItem;
+
+pub(crate) fn without_notification_media(notification: ServerNotification) -> ServerNotification {
+ match notification {
+ ServerNotification::ItemStarted(mut notification) => {
+ notification.item = without_thread_item_media(notification.item);
+ ServerNotification::ItemStarted(notification)
+ }
+ ServerNotification::ItemCompleted(mut notification) => {
+ notification.item = without_thread_item_media(notification.item);
+ ServerNotification::ItemCompleted(notification)
+ }
+ ServerNotification::RawResponseItemCompleted(mut notification) => {
+ match &mut notification.item {
+ ResponseItem::Message { content, .. } => {
+ content.retain(|item| match item {
+ ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false,
+ ContentItem::InputText { .. } | ContentItem::OutputText { .. } => true,
+ });
+ }
+ ResponseItem::FunctionCallOutput { output, .. }
+ | ResponseItem::CustomToolCallOutput { output, .. } => {
+ if let Some(items) = output.content_items_mut() {
+ items.retain(|item| match item {
+ FunctionCallOutputContentItem::InputImage { .. }
+ | FunctionCallOutputContentItem::InputAudio { .. } => false,
+ FunctionCallOutputContentItem::InputText { .. }
+ | FunctionCallOutputContentItem::EncryptedContent { .. } => true,
+ });
+ }
+ }
+ ResponseItem::ImageGenerationCall { result, .. } => result.clear(),
+ ResponseItem::AdditionalTools { .. }
+ | ResponseItem::AgentMessage { .. }
+ | ResponseItem::Reasoning { .. }
+ | ResponseItem::LocalShellCall { .. }
+ | ResponseItem::FunctionCall { .. }
+ | ResponseItem::ToolSearchCall { .. }
+ | ResponseItem::CustomToolCall { .. }
+ | ResponseItem::ToolSearchOutput { .. }
+ | ResponseItem::WebSearchCall { .. }
+ | ResponseItem::Compaction { .. }
+ | ResponseItem::CompactionTrigger { .. }
+ | ResponseItem::ContextCompaction { .. }
+ | ResponseItem::Other => {}
+ }
+ ServerNotification::RawResponseItemCompleted(notification)
+ }
+ ServerNotification::Error(_)
+ | ServerNotification::ThreadStarted(_)
+ | ServerNotification::ThreadStatusChanged(_)
+ | ServerNotification::ThreadArchived(_)
+ | ServerNotification::ThreadDeleted(_)
+ | ServerNotification::ThreadUnarchived(_)
+ | ServerNotification::ThreadClosed(_)
+ | ServerNotification::ThreadReverted(_)
+ | ServerNotification::SkillsChanged(_)
+ | ServerNotification::ThreadNameUpdated(_)
+ | ServerNotification::ThreadGoalUpdated(_)
+ | ServerNotification::ThreadGoalCleared(_)
+ | ServerNotification::ThreadQueueChanged(_)
+ | ServerNotification::ProjectChanged(_)
+ | ServerNotification::ThreadProjectUpdated(_)
+ | ServerNotification::EnvironmentConnected(_)
+ | ServerNotification::EnvironmentDisconnected(_)
+ | ServerNotification::ThreadSettingsUpdated(_)
+ | ServerNotification::ThreadTokenUsageUpdated(_)
+ | ServerNotification::TurnStarted(_)
+ | ServerNotification::HookStarted(_)
+ | ServerNotification::TurnCompleted(_)
+ | ServerNotification::HookCompleted(_)
+ | ServerNotification::TurnDiffUpdated(_)
+ | ServerNotification::TurnPlanUpdated(_)
+ | ServerNotification::ItemGuardianApprovalReviewStarted(_)
+ | ServerNotification::ItemGuardianApprovalReviewCompleted(_)
+ | ServerNotification::StrictReviewRequired(_)
+ | ServerNotification::RawResponseCompleted(_)
+ | ServerNotification::AgentMessageDelta(_)
+ | ServerNotification::PlanDelta(_)
+ | ServerNotification::CommandExecOutputDelta(_)
+ | ServerNotification::ProcessOutputDelta(_)
+ | ServerNotification::ProcessExited(_)
+ | ServerNotification::CommandExecutionOutputDelta(_)
+ | ServerNotification::TerminalInteraction(_)
+ | ServerNotification::FileChangeOutputDelta(_)
+ | ServerNotification::FileChangePatchUpdated(_)
+ | ServerNotification::ServerRequestResolved(_)
+ | ServerNotification::McpToolCallProgress(_)
+ | ServerNotification::McpServerOauthLoginCompleted(_)
+ | ServerNotification::McpServerStatusUpdated(_)
+ | ServerNotification::McpServerEventStream(_)
+ | ServerNotification::AccountUpdated(_)
+ | ServerNotification::AccountRateLimitsUpdated(_)
+ | ServerNotification::AppListUpdated(_)
+ | ServerNotification::RemoteControlStatusChanged(_)
+ | ServerNotification::ExternalAgentConfigImportProgress(_)
+ | ServerNotification::ExternalAgentConfigImportCompleted(_)
+ | ServerNotification::FsChanged(_)
+ | ServerNotification::ReasoningSummaryTextDelta(_)
+ | ServerNotification::ReasoningSummaryPartAdded(_)
+ | ServerNotification::ReasoningTextDelta(_)
+ | ServerNotification::ContextCompacted(_)
+ | ServerNotification::ModelRerouted(_)
+ | ServerNotification::ModelVerification(_)
+ | ServerNotification::TurnModerationMetadata(_)
+ | ServerNotification::ModelSafetyBufferingUpdated(_)
+ | ServerNotification::Warning(_)
+ | ServerNotification::GuardianWarning(_)
+ | ServerNotification::DeprecationNotice(_)
+ | ServerNotification::ConfigWarning(_)
+ | ServerNotification::FuzzyFileSearchSessionUpdated(_)
+ | ServerNotification::FuzzyFileSearchSessionCompleted(_)
+ | ServerNotification::ThreadRealtimeStarted(_)
+ | ServerNotification::ThreadRealtimeItemAdded(_)
+ | ServerNotification::ThreadRealtimeItemStarted(_)
+ | ServerNotification::ThreadRealtimeItemTranscriptDelta(_)
+ | ServerNotification::ThreadRealtimeItemCompleted(_)
+ | ServerNotification::ThreadRealtimeTranscriptDelta(_)
+ | ServerNotification::ThreadRealtimeTranscriptDone(_)
+ | ServerNotification::ThreadRealtimeOutputAudioDelta(_)
+ | ServerNotification::ThreadRealtimeSdp(_)
+ | ServerNotification::ThreadRealtimeError(_)
+ | ServerNotification::ThreadRealtimeClosed(_)
+ | ServerNotification::WindowsWorldWritableWarning(_)
+ | ServerNotification::WindowsSandboxSetupCompleted(_)
+ | ServerNotification::AccountLoginCompleted(_) => notification,
+ }
+}
+
+fn without_thread_item_media(mut item: ThreadItem) -> ThreadItem {
+ match &mut item {
+ ThreadItem::UserMessage { content, .. } => {
+ content.retain(|item| match item {
+ UserInput::Image { .. } | UserInput::Audio { .. } => false,
+ UserInput::Text { .. }
+ | UserInput::LocalImage { .. }
+ | UserInput::LocalAudio { .. }
+ | UserInput::Skill { .. }
+ | UserInput::Mention { .. } => true,
+ });
+ }
+ ThreadItem::DynamicToolCall {
+ content_items: Some(content_items),
+ ..
+ } => {
+ content_items.retain(|item| match item {
+ DynamicToolCallOutputContentItem::InputImage { .. }
+ | DynamicToolCallOutputContentItem::InputAudio { .. } => false,
+ DynamicToolCallOutputContentItem::InputText { .. } => true,
+ });
+ }
+ ThreadItem::McpToolCall {
+ result: Some(result),
+ ..
+ } => {
+ // TODO(ruslan): Handle oversized results that core has already collapsed into a
+ // truncated text preview, which can contain inline media that this filter retains.
+ result.content.retain(|item| {
+ !matches!(
+ item.get("type").and_then(serde_json::Value::as_str),
+ Some("image" | "audio")
+ ) && item
+ .get("resource")
+ .and_then(|resource| resource.get("blob"))
+ .is_none()
+ });
+ }
+ ThreadItem::ImageGeneration(item) => item.result.clear(),
+ ThreadItem::HookPrompt { .. }
+ | ThreadItem::AgentMessage { .. }
+ | ThreadItem::Plan { .. }
+ | ThreadItem::Reasoning { .. }
+ | ThreadItem::CommandExecution { .. }
+ | ThreadItem::FileChange { .. }
+ | ThreadItem::McpToolCall { result: None, .. }
+ | ThreadItem::DynamicToolCall {
+ content_items: None,
+ ..
+ }
+ | ThreadItem::CollabAgentToolCall { .. }
+ | ThreadItem::SubAgentActivity { .. }
+ | ThreadItem::WebSearch(_)
+ | ThreadItem::ImageView { .. }
+ | ThreadItem::Sleep(_)
+ | ThreadItem::EnteredReviewMode { .. }
+ | ThreadItem::ExitedReviewMode { .. }
+ | ThreadItem::ContextCompaction { .. } => {}
+ }
+ item
+}
diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs
index a80036f6a2..3d38ad76ac 100644
--- a/codex-rs/app-server/tests/suite/v2/turn_start.rs
+++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs
@@ -213,6 +213,111 @@ async fn received_response_input_images(server: &wiremock::MockServer) -> Result
Ok(input_images)
}
+#[tokio::test]
+async fn turn_start_omits_notification_media_without_changing_model_input() -> Result<()> {
+ let responses = vec![create_final_assistant_message_sse_response("Done")?];
+ let server = create_mock_responses_server_sequence_unchecked(responses).await;
+
+ let codex_home = TempDir::new()?;
+ MockResponsesConfig::new(&server.uri())
+ .enable_feature(Feature::OmitAppServerNotificationMedia)
+ .write(codex_home.path())?;
+
+ let mut mcp = TestAppServer::builder()
+ .with_codex_home(codex_home.path())
+ .build_initialized()
+ .await?;
+
+ let ThreadStartResponse { thread, .. } = mcp
+ .start_thread(ThreadStartParams {
+ model: Some("mock-model".to_string()),
+ experimental_raw_events: true,
+ ..Default::default()
+ })
+ .await?;
+
+ let _: TurnStartResponse = mcp
+ .request(|request_id| ClientRequest::TurnStart {
+ request_id,
+ params: TurnStartParams {
+ thread_id: thread.id,
+ input: vec![
+ V2UserInput::Text {
+ text: "Describe this image".to_string(),
+ text_elements: Vec::new(),
+ },
+ V2UserInput::Image {
+ url: TINY_PNG_DATA_URL.to_string(),
+ detail: None,
+ },
+ ],
+ ..Default::default()
+ },
+ })
+ .await?;
+
+ let mut user_message_notifications = Vec::new();
+ timeout(DEFAULT_READ_TIMEOUT, async {
+ loop {
+ let notification = mcp
+ .read_stream_until_matching_notification(
+ "item notification or turn completion",
+ |notification| {
+ matches!(
+ notification.method.as_str(),
+ "item/started"
+ | "item/completed"
+ | "rawResponseItem/completed"
+ | "turn/completed"
+ )
+ },
+ )
+ .await?;
+ if notification.method == "turn/completed" {
+ return Ok::<(), anyhow::Error>(());
+ }
+
+ let params = notification.params.context("item notification params")?;
+ let item = &params["item"];
+ let item_type = item["type"].as_str();
+ if item_type == Some("userMessage")
+ || (item_type == Some("message") && item["role"] == "user")
+ {
+ let content = item["content"].as_array().context("user message content")?;
+ if !content
+ .iter()
+ .any(|item| item["text"] == "Describe this image")
+ {
+ continue;
+ }
+ assert_eq!(content.len(), 1);
+ assert!(matches!(
+ content[0]["type"].as_str(),
+ Some("text" | "input_text")
+ ));
+ user_message_notifications.push(notification.method);
+ }
+ }
+ })
+ .await??;
+
+ user_message_notifications.sort();
+ assert_eq!(
+ user_message_notifications,
+ vec![
+ "item/completed",
+ "item/started",
+ "rawResponseItem/completed"
+ ]
+ );
+
+ let model_input_images = received_response_input_images(&server).await?;
+ assert_eq!(model_input_images.len(), 1);
+ assert_eq!(model_input_images[0]["image_url"], TINY_PNG_DATA_URL);
+
+ Ok(())
+}
+
#[tokio::test]
async fn turn_start_with_empty_input_runs_model_request() -> Result<()> {
let responses = vec![create_final_assistant_message_sse_response("Done")?];
diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json
index fe2e5df8ba..1d8a9f4e1f 100644
--- a/codex-rs/core/config.schema.json
+++ b/codex-rs/core/config.schema.json
@@ -808,6 +808,9 @@
"non_prefixed_mcp_tool_names": {
"$ref": "#/definitions/FeatureToml_for_NonPrefixedMcpToolNamesConfigToml"
},
+ "omit_app_server_notification_media": {
+ "type": "boolean"
+ },
"personality": {
"type": "boolean"
},
@@ -6011,6 +6014,9 @@
"non_prefixed_mcp_tool_names": {
"$ref": "#/definitions/FeatureToml_for_NonPrefixedMcpToolNamesConfigToml"
},
+ "omit_app_server_notification_media": {
+ "type": "boolean"
+ },
"personality": {
"type": "boolean"
},
diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs
index 6edf4e627c..cd66f8cd3c 100644
--- a/codex-rs/features/src/lib.rs
+++ b/codex-rs/features/src/lib.rs
@@ -269,6 +269,8 @@ pub enum Feature {
ExternalMigration,
/// Enable extension-backed image generation.
ImageGeneration,
+ /// Omit inline image and audio content from app-server item notifications.
+ OmitAppServerNotificationMedia,
/// Tell the model when a prompt image was resized and include its dimensions.
ImageResizeNotice,
/// Apply one shared pixel and token budget to every image, regardless of legacy detail hints.
@@ -1398,6 +1400,12 @@ pub const FEATURES: &[FeatureSpec] = &[
stage: Stage::Stable,
default_enabled: true,
},
+ FeatureSpec {
+ id: Feature::OmitAppServerNotificationMedia,
+ key: "omit_app_server_notification_media",
+ stage: Stage::UnderDevelopment,
+ default_enabled: false,
+ },
FeatureSpec {
id: Feature::ImageResizeNotice,
key: "image_resize_notice",