Lightweight coding agent that runs in your terminal
Rust
Support image attachments in agents overview background tasks (#44027)
Eric Traut2ce38ae6d84a
diff --git a/codex-rs/tui/src/app/agents_overview.rs b/codex-rs/tui/src/app/agents_overview.rsindex 5df1c79a69..b0ef6e2d19 100644--- a/codex-rs/tui/src/app/agents_overview.rs+++ b/codex-rs/tui/src/app/agents_overview.rs@@ -13,6 +13,7 @@ use crate::bottom_pane::SelectionItem;use crate::bottom_pane::SelectionViewParams;use crate::bottom_pane::popup_consts::standard_popup_hint_line_for_keymap;use crate::chatwidget::ThreadInputStateRestoreMode;+use crate::chatwidget::UserMessage;use codex_app_server_protocol::RequestId;use codex_app_server_protocol::SessionSource;use codex_app_server_protocol::Thread;@@ -20,7 +21,10 @@ use codex_app_server_protocol::ThreadHistoryMode;use codex_app_server_protocol::TurnStartParams;use codex_app_server_protocol::TurnStartResponse;use codex_app_server_protocol::UserInput;+use codex_protocol::models::snapshot_local_user_input;+use codex_protocol::openai_models::InputModality;use codex_protocol::protocol::SubAgentSource;+use codex_protocol::user_input::UserInput as CoreUserInput;pub(crate) const AGENTS_OVERVIEW_VIEW_ID: &str = "agents-overview";@@ -615,7 +619,7 @@ impl App {pub(super) async fn dispatch_agents_overview_task(&mut self,app_server: &mut AppServerSession,- prompt: String,+ prompt: UserMessage,cwd: Option<AbsolutePathBuf>,) {self.refresh_in_memory_config_from_disk_best_effort("starting a background task")@@ -720,6 +724,69 @@ impl App {.or_else(|| self.model_catalog.models.first()).map(|model| model.model.clone());}+ let model = config.model.as_deref().or_else(|| {+ self.model_catalog+ .models+ .iter()+ .find(|model| model.is_default)+ .or_else(|| self.model_catalog.models.first())+ .map(|model| model.model.as_str())+ });+ if !prompt.local_images.is_empty()+ && let Some(model) = model+ && self.model_catalog.models.iter().any(|preset| {+ preset.model == model && !preset.input_modalities.contains(&InputModality::Image)+ })+ {+ let message = format!(+ "Model {model} does not support image inputs. Remove images or switch models."+ );+ self.restore_agents_overview_prompt(prompt);+ self.chat_widget.add_error_message(message);+ return;+ }+ let images = prompt+ .local_images+ .iter()+ .map(|image| {+ Ok(CoreUserInput::LocalImage {+ path: std::path::absolute(&image.path)?,+ detail: None,+ })+ })+ .collect::<std::io::Result<Vec<_>>>();+ let images = match images {+ Ok(images) => images,+ Err(error) => {+ self.restore_agents_overview_prompt(prompt);+ self.chat_widget+ .add_error_message(format!("Failed to prepare image: {error}"));+ return;+ }+ };+ let images = if app_server.uses_remote_workspace() && !images.is_empty() {+ match tokio::task::spawn_blocking(move || {+ let mut images = images;+ for image in &mut images {+ snapshot_local_user_input(image)?;+ }+ Ok::<_, std::io::Error>(images)+ })+ .await+ .map_err(std::io::Error::other)+ .and_then(|result| result)+ {+ Ok(images) => images,+ Err(error) => {+ self.restore_agents_overview_prompt(prompt);+ self.chat_widget+ .add_error_message(format!("Failed to prepare image: {error}"));+ return;+ }+ }+ } else {+ images+ };match app_server.start_thread_with_session_start_source(&self.local_settings,@@ -735,8 +802,13 @@ impl App {self.agents_overview.dispatched_requests.insert(thread_id, Vec::new());- self.submit_agents_overview_prompt(app_server, thread_id, prompt)- .await;+ self.submit_agents_overview_prompt(+ app_server,+ thread_id,+ prompt,+ images.into_iter().map(Into::into).collect(),+ )+ .await;}Err(error) => {self.restore_agents_overview_prompt(prompt);@@ -750,18 +822,27 @@ impl App {&mut self,app_server: &AppServerSession,thread_id: ThreadId,- prompt: String,+ prompt: UserMessage,+ mut input: Vec<UserInput>,) {+ if !prompt.text.is_empty() {+ input.push(UserInput::Text {+ text: prompt.text.clone(),+ text_elements: prompt+ .text_elements+ .iter()+ .cloned()+ .map(Into::into)+ .collect(),+ });+ }let result = app_server.request_handle().request_typed::<TurnStartResponse>(ClientRequest::TurnStart {request_id: RequestId::String(Uuid::new_v4().to_string()),params: TurnStartParams {thread_id: thread_id.to_string(),- input: vec![UserInput::Text {- text: prompt.clone(),- text_elements: Vec::new(),- }],+ input,..Default::default()},})diff --git a/codex-rs/tui/src/app/agents_overview_composer.rs b/codex-rs/tui/src/app/agents_overview_composer.rsindex 7a15da32f0..5d482a9ea4 100644--- a/codex-rs/tui/src/app/agents_overview_composer.rs+++ b/codex-rs/tui/src/app/agents_overview_composer.rs@@ -4,6 +4,7 @@use super::*;use crate::bottom_pane::ChatComposer;use crate::bottom_pane::ChatComposerConfig;+use crate::chatwidget::UserMessage;impl App {pub(super) fn sync_agents_overview_composer(&self) {@@ -21,6 +22,7 @@ impl App {self.config.disable_paste_burst,ChatComposerConfig {trim_submission: false,+ image_paste_enabled: true,..ChatComposerConfig::plain_text()},);@@ -39,18 +41,37 @@ impl App {composer.set_keymap_bindings(&self.keymap);}- pub(super) fn restore_agents_overview_prompt(&mut self, prompt: String) {+ pub(super) fn restore_agents_overview_prompt(&mut self, prompt: UserMessage) {if let Ok(mut state) = self.agents_overview.view_state.lock()&& let Some(composer) = state.composer.as_mut()&& composer.is_empty()&& !composer.is_in_paste_burst()&& !composer.popup_active(){- composer.set_text_content(prompt, Vec::new(), Vec::new());+ composer.set_text_content(+ prompt.text,+ prompt.text_elements,+ prompt+ .local_images+ .into_iter()+ .map(|image| image.path)+ .collect(),+ );composer.move_cursor_to_end();return;}+ let hint = (!prompt.local_images.is_empty()).then(|| {+ format!(+ "Reattach images from: {}",+ prompt+ .local_images+ .iter()+ .map(|image| image.path.display().to_string())+ .collect::<Vec<_>>()+ .join(", ")+ )+ });self.chat_widget- .add_info_message(format!("Unsent task: {prompt}"), /*hint*/ None);+ .add_info_message(format!("Unsent task: {}", prompt.text), hint);}}diff --git a/codex-rs/tui/src/app/agents_overview_input.rs b/codex-rs/tui/src/app/agents_overview_input.rsindex 9be8c33766..c2a194a4dd 100644--- a/codex-rs/tui/src/app/agents_overview_input.rs+++ b/codex-rs/tui/src/app/agents_overview_input.rs@@ -3,6 +3,10 @@use super::*;use crate::bottom_pane::InputResult;+use crate::chatwidget::UserMessage;+use crate::clipboard_paste::paste_image_to_temp_png;+use crossterm::event::KeyEventKind;+use crossterm::event::KeyModifiers;impl AgentsOverviewView {pub(super) fn handle_composer_key(&mut self, key: KeyEvent) {@@ -16,6 +20,20 @@ impl AgentsOverviewView {let Some(composer) = state.composer.as_mut() else {return;};+ if key.kind == KeyEventKind::Press+ && matches!(key.code, KeyCode::Char('v' | 'V'))+ && key+ .modifiers+ .intersects(KeyModifiers::CONTROL | KeyModifiers::ALT)+ {+ match paste_image_to_temp_png() {+ Ok((path, _)) => composer.attach_image(path),+ Err(error) => self.app_event_tx.send(AppEvent::InsertHistoryCell(Box::new(+ crate::history_cell::new_error_event(format!("Failed to paste image: {error}")),+ ))),+ }+ return;+ }if offline&& !composer.popup_active()&& (self.composer_keymap.submit.is_pressed(key)@@ -27,11 +45,26 @@ impl AgentsOverviewView {return;}let (result, _) = composer.handle_key_event(key);+ let prompt = if let InputResult::Submitted {+ text,+ text_elements,+ } = result+ {+ Some(UserMessage {+ text,+ text_elements,+ local_images: composer.take_recent_submission_images_with_placeholders(),+ remote_image_urls: Vec::new(),+ mention_bindings: Vec::new(),+ })+ } else {+ None+ };drop(state);- if let InputResult::Submitted { text, .. } = result {+ if let Some(prompt) = prompt {self.app_event_tx.send(AppEvent::DispatchAgentsOverviewTask {- prompt: text,+ prompt,cwd: (!status_grouping).then(|| self.selected_row().map(|row| row.thread.cwd.clone())).flatten(),diff --git a/codex-rs/tui/src/app/agents_overview_tests.rs b/codex-rs/tui/src/app/agents_overview_tests.rsindex c7e9f4def2..6dfe993cb7 100644--- a/codex-rs/tui/src/app/agents_overview_tests.rs+++ b/codex-rs/tui/src/app/agents_overview_tests.rs@@ -145,7 +145,7 @@ async fn overview_composer_preserves_editing_and_routes_focus() {view.handle_key_event(KeyCode::Esc.into());view.handle_key_event(KeyCode::Tab.into());assert!(- matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt == expected.0)+ matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt.text == expected.0));view.handle_key_event(KeyCode::Up.into());assert_eq!(overview_draft(&app).0, expected.0);@@ -161,7 +161,7 @@ async fn overview_composer_preserves_editing_and_routes_focus() {app.dispatch_agents_overview_task(&mut server, "retry me".into(), Some(app.config.cwd.clone())).await;view.handle_paste(" later".into());- app.submit_agents_overview_prompt(&server, thread_id, "older failure".into())+ app.submit_agents_overview_prompt(&server, thread_id, "older failure".into(), Vec::new()).await;assert_eq!(overview_draft(&app).0, "retry me later");server.shutdown().await.unwrap();@@ -222,7 +222,7 @@ async fn overview_composer_preserves_pastes_and_editor_bindings() {view.handle_key_event(KeyCode::Char('x').into());view.handle_key_event(KeyCode::F(8).into());assert!(- matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt == format!("{pasted}\nabcxlast line"))+ matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt.text == format!("{pasted}\nabcxlast line")));}@@ -1094,7 +1094,7 @@ async fn shared_overview_shows_only_root_sessions() {assert!(matches!(event_rx.try_recv(),Ok(AppEvent::DispatchAgentsOverviewTask { prompt, cwd: None })- if prompt == "Use the current project"+ if prompt.text == "Use the current project"));action_view.handle_key_event(KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE));action_view.handle_key_event(KeyEvent::new(KeyCode::Char('s'), KeyModifiers::CONTROL));@@ -1114,7 +1114,7 @@ async fn shared_overview_shows_only_root_sessions() {assert!(matches!(event_rx.try_recv(),Ok(AppEvent::DispatchAgentsOverviewTask { prompt, cwd: Some(cwd) })- if prompt == "Fix the flaky tests after all retries complete"+ if prompt.text == "Fix the flaky tests after all retries complete"&& cwd == test_path_buf("/tmp/project").abs()));assert!(action_view.handle_paste(" ".to_string()));@@ -1131,7 +1131,7 @@ async fn shared_overview_shows_only_root_sessions() {assert!(matches!(event_rx.try_recv(),Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. })- if prompt.trim() == "Continue working"+ if prompt.text.trim() == "Continue working"));action_view.handle_key_event(KeyEvent::new(KeyCode::Char('n'), KeyModifiers::CONTROL));action_view.handle_paste("First line\nSecond line".into());diff --git a/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs b/codex-rs/tui/src/app/tests/background_task_defaults_tests.rsindex 504f0f7c84..d3a9f24fa2 100644--- a/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs+++ b/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs@@ -5,10 +5,220 @@ use crate::app::agents_overview::AGENTS_OVERVIEW_VIEW_ID;use crate::app::tests::session_lifecycle_requests::HistoryCapabilities;use crate::app::tests::session_lifecycle_requests::recorded_params;use crate::app::tests::session_lifecycle_requests::start_recording_app_server_with_history;+use crate::bottom_pane::BottomPaneView;+use crate::bottom_pane::LocalImageAttachment;+use crate::chatwidget::UserMessage;+use crate::chatwidget::tests::helpers::render_bottom_popup;+use crate::model_catalog::ModelCatalog;use crate::test_support::PathBufExt;+use codex_app_server_protocol::UserInput;+use codex_protocol::openai_models::InputModality;use codex_state::SqliteConfig;+use crossterm::event::KeyCode;use pretty_assertions::assert_eq;+#[tokio::test]+async fn background_task_sends_pasted_image_with_first_prompt() -> Result<()> {+ let (mut app, mut events, _) = make_test_app_with_channels().await;+ let image_dir = tempdir()?;+ let image_path = image_dir.path().join("pasted.png");+ image::RgbImage::new(1, 1).save(&image_path)?;+ let pasted_path = pathdiff::diff_paths(&image_path, std::env::current_dir()?)+ .unwrap_or_else(|| image_path.clone());+ let mut view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);+ assert!(view.handle_paste(pasted_path.to_string_lossy().into_owned()));+ assert!(view.handle_paste("Describe this".into()));+ let rendered_view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);+ app.chat_widget+ .show_bottom_pane_view(Box::new(rendered_view));+ insta::assert_snapshot!(+ render_bottom_popup(&app.chat_widget, /*width*/ 80)+ .lines()+ .find(|line| line.contains("[Image #1]"))+ .expect("image attachment visible"),+ @"› [Image #1] Describe this"+ );+ view.handle_key_event(KeyCode::Enter.into());+ let prompt = match events.try_recv()? {+ AppEvent::DispatchAgentsOverviewTask { prompt, .. } => prompt,+ event => panic!("expected task dispatch, got {event:?}"),+ };+ assert_eq!(prompt.local_images[0].path, pasted_path);+ assert_eq!(+ prompt.text_elements[0].placeholder(&prompt.text),+ Some("[Image #1]")+ );++ let (mut server, requests, proxy) = start_recording_app_server_with_history(+ &app.config,+ HistoryCapabilities::Current,+ /*blocked_thread_list*/ None,+ /*failed_thread_name*/ None,+ crate::app_server_session::ThreadParamsMode::Embedded,+ LoaderOverrides::default(),+ )+ .await?;+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), /*cwd*/ None)+ .await;+ let turns = recorded_params(&requests, "turn/start");+ assert_eq!(turns.len(), 1);+ assert_eq!(+ turns[0]["input"],+ serde_json::to_value(vec![+ UserInput::LocalImage {+ path: std::path::absolute(&pasted_path)?,+ detail: None+ },+ UserInput::Text {+ text: prompt.text.clone(),+ text_elements: prompt+ .text_elements+ .iter()+ .cloned()+ .map(Into::into)+ .collect(),+ },+ ])?+ );++ app.cli_kv_overrides = vec![("model".into(), toml::Value::Integer(1))];+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), Some(app.config.cwd.clone()))+ .await;+ let restored = app+ .agents_overview+ .view_state+ .lock()+ .unwrap()+ .composer+ .as_ref()+ .unwrap()+ .draft_snapshot();+ assert_eq!(+ (restored.text, restored.text_elements, restored.local_images),+ (+ prompt.text.clone(),+ prompt.text_elements.clone(),+ prompt.local_images.clone()+ )+ );+ view.handle_key_event(KeyCode::Enter.into());+ assert!(view.handle_paste("A newer draft".into()));+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), Some(app.config.cwd.clone()))+ .await;+ let current = app+ .agents_overview+ .view_state+ .lock()+ .unwrap()+ .composer+ .as_ref()+ .unwrap()+ .draft_snapshot();+ assert_eq!(current.text, "A newer draft");+ let notice = std::iter::from_fn(|| events.try_recv().ok())+ .filter_map(|event| match event {+ AppEvent::InsertHistoryCell(cell) => {+ Some(lines_to_single_string(&cell.display_lines(/*width*/ 200)))+ }+ _ => None,+ })+ .find(|message| message.contains("Reattach images from:"))+ .expect("attachment recovery notice");+ assert!(notice.contains(&pasted_path.display().to_string()));+ server.shutdown().await?;+ proxy.await??;+ Ok(())+}++#[tokio::test]+async fn remote_background_task_sends_clipboard_image_bytes() -> Result<()> {+ let mut app = make_test_app_with_channels().await.0;+ let image_dir = tempdir()?;+ let image_path = image_dir.path().join("pasted.png");+ image::RgbImage::new(1, 1).save(&image_path)?;+ let mut prompt = UserMessage::from("Describe this image");+ prompt.local_images.push(LocalImageAttachment {+ placeholder: "[Image #1]".into(),+ path: image_path,+ });+ let (mut server, requests, proxy) = start_recording_app_server_with_history(+ &app.config,+ HistoryCapabilities::Current,+ /*blocked_thread_list*/ None,+ /*failed_thread_name*/ None,+ crate::app_server_session::ThreadParamsMode::Remote,+ LoaderOverrides::default(),+ )+ .await?;+ app.dispatch_agents_overview_task(&mut server, prompt, /*cwd*/ None)+ .await;+ let turns = recorded_params(&requests, "turn/start");+ assert_eq!(turns.len(), 1);+ let image = &turns[0]["input"][0];+ assert_eq!(image["type"], "image");+ assert!(+ image["url"]+ .as_str()+ .is_some_and(|url| url.starts_with("data:image/png;base64,"))+ );+ assert_eq!(turns[0]["input"][1]["text"], "Describe this image");+ server.shutdown().await?;+ proxy.await??;+ Ok(())+}++#[tokio::test]+async fn background_task_rejects_images_for_text_only_model() -> Result<()> {+ let (mut app, mut events, _) = make_test_app_with_channels().await;+ let mut preset = app.model_catalog.models[0].clone();+ preset.model = "text-only-test-model".into();+ preset.input_modalities = vec![InputModality::Text];+ app.model_catalog = std::sync::Arc::new(ModelCatalog::new(vec![preset]));+ app.harness_overrides.model = Some("text-only-test-model".into());+ let view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);+ app.chat_widget.show_bottom_pane_view(Box::new(view));+ let mut prompt = UserMessage::from("[Image #1] Describe this");+ prompt.local_images.push(LocalImageAttachment {+ placeholder: "[Image #1]".into(),+ path: "/tmp/test-pasted.png".into(),+ });+ let (mut server, requests, proxy) = start_recording_app_server_with_history(+ &app.config,+ HistoryCapabilities::Current,+ /*blocked_thread_list*/ None,+ /*failed_thread_name*/ None,+ crate::app_server_session::ThreadParamsMode::Embedded,+ LoaderOverrides::default(),+ )+ .await?;+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), /*cwd*/ None)+ .await;+ assert!(recorded_params(&requests, "thread/start").is_empty());+ let draft = app+ .agents_overview+ .view_state+ .lock()+ .unwrap()+ .composer+ .as_ref()+ .unwrap()+ .draft_snapshot();+ assert_eq!(draft.local_images, prompt.local_images);+ let error = std::iter::from_fn(|| events.try_recv().ok())+ .filter_map(|event| match event {+ AppEvent::InsertHistoryCell(cell) => {+ Some(lines_to_single_string(&cell.display_lines(/*width*/ 100)))+ }+ _ => None,+ })+ .find(|message| message.contains("does not support image inputs"))+ .expect("visible model rejection");+ insta::assert_snapshot!(error, @"■ Model text-only-test-model does not support image inputs. Remove images or switch models.");+ server.shutdown().await?;+ proxy.await??;+ Ok(())+}+#[tokio::test]async fn background_task_reads_server_defaults_for_actual_destination() -> Result<()> {for (mode, explicit_cwd, launch_override, expected_cwd, expected_model) in [diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rsindex 9ad0b0b034..c397c98f64 100644--- a/codex-rs/tui/src/app_event.rs+++ b/codex-rs/tui/src/app_event.rs@@ -278,7 +278,7 @@ pub(crate) enum AppEvent {},/// Start a background task directly from the shared dashboard.DispatchAgentsOverviewTask {- prompt: String,+ prompt: UserMessage,cwd: Option<AbsolutePathBuf>,},/// Rename a task directly from the shared dashboard.