Lightweight coding agent that runs in your terminal
Rust
Add app-server notification media filtering (#41416)
Vivian Fang03147407e3a0
diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rsindex d06938d781..7fcfeb8d35 100644--- a/codex-rs/app-server/src/bespoke_event_handling.rs+++ b/codex-rs/app-server/src/bespoke_event_handling.rs@@ -1,5 +1,6 @@use crate::error_code::internal_error;use crate::error_code::invalid_request;+use crate::notification_media::without_notification_media;use crate::outgoing_message::ClientRequestResult;use crate::outgoing_message::ThreadScopedOutgoingMessageSender;use crate::request_processors::populate_thread_turns_from_history;@@ -91,6 +92,7 @@ use codex_app_server_protocol::guardian_auto_approval_review_notification;use codex_app_server_protocol::item_event_to_server_notification;use codex_core::CodexThread;use codex_core::ThreadManager;+use codex_features::Feature;use codex_guardian_v2::StrictReviewReason;use codex_protocol::ThreadId;use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool;@@ -1068,11 +1070,14 @@ pub(crate) async fn apply_bespoke_event_handling(_ => None,};if should_emit {- let notification = item_event_to_server_notification(+ let mut notification = item_event_to_server_notification(EventMsg::ItemStarted(event),&conversation_id.to_string(),&event_turn_id,);+ if conversation.enabled(Feature::OmitAppServerNotificationMedia) {+ notification = without_notification_media(notification);+ }outgoing.send_server_notification(notification).await;}if let Some(params) = dynamic_tool_call_params {@@ -1093,11 +1098,14 @@ pub(crate) async fn apply_bespoke_event_handling(&event.item,).await;- let notification = item_event_to_server_notification(+ let mut notification = item_event_to_server_notification(EventMsg::ItemCompleted(event),&conversation_id.to_string(),&event_turn_id,);+ if conversation.enabled(Feature::OmitAppServerNotificationMedia) {+ notification = without_notification_media(notification);+ }outgoing.send_server_notification(notification).await;}msg @ (EventMsg::PatchApplyUpdated(_) | EventMsg::TerminalInteraction(_)) => {@@ -1129,13 +1137,17 @@ pub(crate) async fn apply_bespoke_event_handling(.await;}EventMsg::RawResponseItem(raw_response_item_event) => {- maybe_emit_raw_response_item_completed(- conversation_id,- &event_turn_id,- raw_response_item_event.item,- &outgoing,- )- .await;+ let mut notification = ServerNotification::RawResponseItemCompleted(+ RawResponseItemCompletedNotification {+ thread_id: conversation_id.to_string(),+ turn_id: event_turn_id,+ item: raw_response_item_event.item,+ },+ );+ if conversation.enabled(Feature::OmitAppServerNotificationMedia) {+ notification = without_notification_media(notification);+ }+ outgoing.send_server_notification(notification).await;}EventMsg::RawResponseCompleted(raw_response_completed_event) => {let notification = RawResponseCompletedNotification {@@ -1512,22 +1524,6 @@ async fn complete_command_execution_item(.await;}-async fn maybe_emit_raw_response_item_completed(- conversation_id: ThreadId,- turn_id: &str,- item: codex_protocol::models::ResponseItem,- outgoing: &ThreadScopedOutgoingMessageSender,-) {- let notification = RawResponseItemCompletedNotification {- thread_id: conversation_id.to_string(),- turn_id: turn_id.to_string(),- item,- };- outgoing- .send_server_notification(ServerNotification::RawResponseItemCompleted(notification))- .await;-}-async fn find_and_remove_turn_summary(_conversation_id: ThreadId,thread_state: &Arc<Mutex<ThreadState>>,diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rsindex 6aa20e574d..0499145122 100644--- a/codex-rs/app-server/src/lib.rs+++ b/codex-rs/app-server/src/lib.rs@@ -119,6 +119,7 @@ mod mcp_refresh;mod message_processor;mod models;mod models_refresh_worker;+mod notification_media;mod otel_reloader;mod outgoing_message;mod realtime_event_handling;diff --git a/codex-rs/app-server/src/notification_media.rs b/codex-rs/app-server/src/notification_media.rsnew file mode 100644index 0000000000..f6e291bf7f--- /dev/null+++ b/codex-rs/app-server/src/notification_media.rs@@ -0,0 +1,196 @@+use codex_app_server_protocol::DynamicToolCallOutputContentItem;+use codex_app_server_protocol::ServerNotification;+use codex_app_server_protocol::ThreadItem;+use codex_app_server_protocol::UserInput;+use codex_protocol::models::ContentItem;+use codex_protocol::models::FunctionCallOutputContentItem;+use codex_protocol::models::ResponseItem;++pub(crate) fn without_notification_media(notification: ServerNotification) -> ServerNotification {+ match notification {+ ServerNotification::ItemStarted(mut notification) => {+ notification.item = without_thread_item_media(notification.item);+ ServerNotification::ItemStarted(notification)+ }+ ServerNotification::ItemCompleted(mut notification) => {+ notification.item = without_thread_item_media(notification.item);+ ServerNotification::ItemCompleted(notification)+ }+ ServerNotification::RawResponseItemCompleted(mut notification) => {+ match &mut notification.item {+ ResponseItem::Message { content, .. } => {+ content.retain(|item| match item {+ ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false,+ ContentItem::InputText { .. } | ContentItem::OutputText { .. } => true,+ });+ }+ ResponseItem::FunctionCallOutput { output, .. }+ | ResponseItem::CustomToolCallOutput { output, .. } => {+ if let Some(items) = output.content_items_mut() {+ items.retain(|item| match item {+ FunctionCallOutputContentItem::InputImage { .. }+ | FunctionCallOutputContentItem::InputAudio { .. } => false,+ FunctionCallOutputContentItem::InputText { .. }+ | FunctionCallOutputContentItem::EncryptedContent { .. } => true,+ });+ }+ }+ ResponseItem::ImageGenerationCall { result, .. } => result.clear(),+ ResponseItem::AdditionalTools { .. }+ | ResponseItem::AgentMessage { .. }+ | ResponseItem::Reasoning { .. }+ | ResponseItem::LocalShellCall { .. }+ | ResponseItem::FunctionCall { .. }+ | ResponseItem::ToolSearchCall { .. }+ | ResponseItem::CustomToolCall { .. }+ | ResponseItem::ToolSearchOutput { .. }+ | ResponseItem::WebSearchCall { .. }+ | ResponseItem::Compaction { .. }+ | ResponseItem::CompactionTrigger { .. }+ | ResponseItem::ContextCompaction { .. }+ | ResponseItem::Other => {}+ }+ ServerNotification::RawResponseItemCompleted(notification)+ }+ ServerNotification::Error(_)+ | ServerNotification::ThreadStarted(_)+ | ServerNotification::ThreadStatusChanged(_)+ | ServerNotification::ThreadArchived(_)+ | ServerNotification::ThreadDeleted(_)+ | ServerNotification::ThreadUnarchived(_)+ | ServerNotification::ThreadClosed(_)+ | ServerNotification::ThreadReverted(_)+ | ServerNotification::SkillsChanged(_)+ | ServerNotification::ThreadNameUpdated(_)+ | ServerNotification::ThreadGoalUpdated(_)+ | ServerNotification::ThreadGoalCleared(_)+ | ServerNotification::ThreadQueueChanged(_)+ | ServerNotification::ProjectChanged(_)+ | ServerNotification::ThreadProjectUpdated(_)+ | ServerNotification::EnvironmentConnected(_)+ | ServerNotification::EnvironmentDisconnected(_)+ | ServerNotification::ThreadSettingsUpdated(_)+ | ServerNotification::ThreadTokenUsageUpdated(_)+ | ServerNotification::TurnStarted(_)+ | ServerNotification::HookStarted(_)+ | ServerNotification::TurnCompleted(_)+ | ServerNotification::HookCompleted(_)+ | ServerNotification::TurnDiffUpdated(_)+ | ServerNotification::TurnPlanUpdated(_)+ | ServerNotification::ItemGuardianApprovalReviewStarted(_)+ | ServerNotification::ItemGuardianApprovalReviewCompleted(_)+ | ServerNotification::StrictReviewRequired(_)+ | ServerNotification::RawResponseCompleted(_)+ | ServerNotification::AgentMessageDelta(_)+ | ServerNotification::PlanDelta(_)+ | ServerNotification::CommandExecOutputDelta(_)+ | ServerNotification::ProcessOutputDelta(_)+ | ServerNotification::ProcessExited(_)+ | ServerNotification::CommandExecutionOutputDelta(_)+ | ServerNotification::TerminalInteraction(_)+ | ServerNotification::FileChangeOutputDelta(_)+ | ServerNotification::FileChangePatchUpdated(_)+ | ServerNotification::ServerRequestResolved(_)+ | ServerNotification::McpToolCallProgress(_)+ | ServerNotification::McpServerOauthLoginCompleted(_)+ | ServerNotification::McpServerStatusUpdated(_)+ | ServerNotification::McpServerEventStream(_)+ | ServerNotification::AccountUpdated(_)+ | ServerNotification::AccountRateLimitsUpdated(_)+ | ServerNotification::AppListUpdated(_)+ | ServerNotification::RemoteControlStatusChanged(_)+ | ServerNotification::ExternalAgentConfigImportProgress(_)+ | ServerNotification::ExternalAgentConfigImportCompleted(_)+ | ServerNotification::FsChanged(_)+ | ServerNotification::ReasoningSummaryTextDelta(_)+ | ServerNotification::ReasoningSummaryPartAdded(_)+ | ServerNotification::ReasoningTextDelta(_)+ | ServerNotification::ContextCompacted(_)+ | ServerNotification::ModelRerouted(_)+ | ServerNotification::ModelVerification(_)+ | ServerNotification::TurnModerationMetadata(_)+ | ServerNotification::ModelSafetyBufferingUpdated(_)+ | ServerNotification::Warning(_)+ | ServerNotification::GuardianWarning(_)+ | ServerNotification::DeprecationNotice(_)+ | ServerNotification::ConfigWarning(_)+ | ServerNotification::FuzzyFileSearchSessionUpdated(_)+ | ServerNotification::FuzzyFileSearchSessionCompleted(_)+ | ServerNotification::ThreadRealtimeStarted(_)+ | ServerNotification::ThreadRealtimeItemAdded(_)+ | ServerNotification::ThreadRealtimeItemStarted(_)+ | ServerNotification::ThreadRealtimeItemTranscriptDelta(_)+ | ServerNotification::ThreadRealtimeItemCompleted(_)+ | ServerNotification::ThreadRealtimeTranscriptDelta(_)+ | ServerNotification::ThreadRealtimeTranscriptDone(_)+ | ServerNotification::ThreadRealtimeOutputAudioDelta(_)+ | ServerNotification::ThreadRealtimeSdp(_)+ | ServerNotification::ThreadRealtimeError(_)+ | ServerNotification::ThreadRealtimeClosed(_)+ | ServerNotification::WindowsWorldWritableWarning(_)+ | ServerNotification::WindowsSandboxSetupCompleted(_)+ | ServerNotification::AccountLoginCompleted(_) => notification,+ }+}++fn without_thread_item_media(mut item: ThreadItem) -> ThreadItem {+ match &mut item {+ ThreadItem::UserMessage { content, .. } => {+ content.retain(|item| match item {+ UserInput::Image { .. } | UserInput::Audio { .. } => false,+ UserInput::Text { .. }+ | UserInput::LocalImage { .. }+ | UserInput::LocalAudio { .. }+ | UserInput::Skill { .. }+ | UserInput::Mention { .. } => true,+ });+ }+ ThreadItem::DynamicToolCall {+ content_items: Some(content_items),+ ..+ } => {+ content_items.retain(|item| match item {+ DynamicToolCallOutputContentItem::InputImage { .. }+ | DynamicToolCallOutputContentItem::InputAudio { .. } => false,+ DynamicToolCallOutputContentItem::InputText { .. } => true,+ });+ }+ ThreadItem::McpToolCall {+ result: Some(result),+ ..+ } => {+ // TODO(ruslan): Handle oversized results that core has already collapsed into a+ // truncated text preview, which can contain inline media that this filter retains.+ result.content.retain(|item| {+ !matches!(+ item.get("type").and_then(serde_json::Value::as_str),+ Some("image" | "audio")+ ) && item+ .get("resource")+ .and_then(|resource| resource.get("blob"))+ .is_none()+ });+ }+ ThreadItem::ImageGeneration(item) => item.result.clear(),+ ThreadItem::HookPrompt { .. }+ | ThreadItem::AgentMessage { .. }+ | ThreadItem::Plan { .. }+ | ThreadItem::Reasoning { .. }+ | ThreadItem::CommandExecution { .. }+ | ThreadItem::FileChange { .. }+ | ThreadItem::McpToolCall { result: None, .. }+ | ThreadItem::DynamicToolCall {+ content_items: None,+ ..+ }+ | ThreadItem::CollabAgentToolCall { .. }+ | ThreadItem::SubAgentActivity { .. }+ | ThreadItem::WebSearch(_)+ | ThreadItem::ImageView { .. }+ | ThreadItem::Sleep(_)+ | ThreadItem::EnteredReviewMode { .. }+ | ThreadItem::ExitedReviewMode { .. }+ | ThreadItem::ContextCompaction { .. } => {}+ }+ item+}diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rsindex a80036f6a2..3d38ad76ac 100644--- a/codex-rs/app-server/tests/suite/v2/turn_start.rs+++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs@@ -213,6 +213,111 @@ async fn received_response_input_images(server: &wiremock::MockServer) -> ResultOk(input_images)}+#[tokio::test]+async fn turn_start_omits_notification_media_without_changing_model_input() -> Result<()> {+ let responses = vec![create_final_assistant_message_sse_response("Done")?];+ let server = create_mock_responses_server_sequence_unchecked(responses).await;++ let codex_home = TempDir::new()?;+ MockResponsesConfig::new(&server.uri())+ .enable_feature(Feature::OmitAppServerNotificationMedia)+ .write(codex_home.path())?;++ let mut mcp = TestAppServer::builder()+ .with_codex_home(codex_home.path())+ .build_initialized()+ .await?;++ let ThreadStartResponse { thread, .. } = mcp+ .start_thread(ThreadStartParams {+ model: Some("mock-model".to_string()),+ experimental_raw_events: true,+ ..Default::default()+ })+ .await?;++ let _: TurnStartResponse = mcp+ .request(|request_id| ClientRequest::TurnStart {+ request_id,+ params: TurnStartParams {+ thread_id: thread.id,+ input: vec![+ V2UserInput::Text {+ text: "Describe this image".to_string(),+ text_elements: Vec::new(),+ },+ V2UserInput::Image {+ url: TINY_PNG_DATA_URL.to_string(),+ detail: None,+ },+ ],+ ..Default::default()+ },+ })+ .await?;++ let mut user_message_notifications = Vec::new();+ timeout(DEFAULT_READ_TIMEOUT, async {+ loop {+ let notification = mcp+ .read_stream_until_matching_notification(+ "item notification or turn completion",+ |notification| {+ matches!(+ notification.method.as_str(),+ "item/started"+ | "item/completed"+ | "rawResponseItem/completed"+ | "turn/completed"+ )+ },+ )+ .await?;+ if notification.method == "turn/completed" {+ return Ok::<(), anyhow::Error>(());+ }++ let params = notification.params.context("item notification params")?;+ let item = ¶ms["item"];+ let item_type = item["type"].as_str();+ if item_type == Some("userMessage")+ || (item_type == Some("message") && item["role"] == "user")+ {+ let content = item["content"].as_array().context("user message content")?;+ if !content+ .iter()+ .any(|item| item["text"] == "Describe this image")+ {+ continue;+ }+ assert_eq!(content.len(), 1);+ assert!(matches!(+ content[0]["type"].as_str(),+ Some("text" | "input_text")+ ));+ user_message_notifications.push(notification.method);+ }+ }+ })+ .await??;++ user_message_notifications.sort();+ assert_eq!(+ user_message_notifications,+ vec![+ "item/completed",+ "item/started",+ "rawResponseItem/completed"+ ]+ );++ let model_input_images = received_response_input_images(&server).await?;+ assert_eq!(model_input_images.len(), 1);+ assert_eq!(model_input_images[0]["image_url"], TINY_PNG_DATA_URL);++ Ok(())+}+#[tokio::test]async fn turn_start_with_empty_input_runs_model_request() -> Result<()> {let responses = vec![create_final_assistant_message_sse_response("Done")?];diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.jsonindex fe2e5df8ba..1d8a9f4e1f 100644--- a/codex-rs/core/config.schema.json+++ b/codex-rs/core/config.schema.json@@ -808,6 +808,9 @@"non_prefixed_mcp_tool_names": {"$ref": "#/definitions/FeatureToml_for_NonPrefixedMcpToolNamesConfigToml"},+ "omit_app_server_notification_media": {+ "type": "boolean"+ },"personality": {"type": "boolean"},@@ -6011,6 +6014,9 @@"non_prefixed_mcp_tool_names": {"$ref": "#/definitions/FeatureToml_for_NonPrefixedMcpToolNamesConfigToml"},+ "omit_app_server_notification_media": {+ "type": "boolean"+ },"personality": {"type": "boolean"},diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rsindex 6edf4e627c..cd66f8cd3c 100644--- a/codex-rs/features/src/lib.rs+++ b/codex-rs/features/src/lib.rs@@ -269,6 +269,8 @@ pub enum Feature {ExternalMigration,/// Enable extension-backed image generation.ImageGeneration,+ /// Omit inline image and audio content from app-server item notifications.+ OmitAppServerNotificationMedia,/// Tell the model when a prompt image was resized and include its dimensions.ImageResizeNotice,/// Apply one shared pixel and token budget to every image, regardless of legacy detail hints.@@ -1398,6 +1400,12 @@ pub const FEATURES: &[FeatureSpec] = &[stage: Stage::Stable,default_enabled: true,},+ FeatureSpec {+ id: Feature::OmitAppServerNotificationMedia,+ key: "omit_app_server_notification_media",+ stage: Stage::UnderDevelopment,+ default_enabled: false,+ },FeatureSpec {id: Feature::ImageResizeNotice,key: "image_resize_notice",