Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Support image attachments in agents overview background tasks (#44027)

Eric Traut2ce38ae6d84a
diff --git a/codex-rs/tui/src/app/agents_overview.rs b/codex-rs/tui/src/app/agents_overview.rs
index 5df1c79a69..b0ef6e2d19 100644
--- a/codex-rs/tui/src/app/agents_overview.rs
+++ b/codex-rs/tui/src/app/agents_overview.rs
@@ -13,6 +13,7 @@ use crate::bottom_pane::SelectionItem;
use crate::bottom_pane::SelectionViewParams;
use crate::bottom_pane::popup_consts::standard_popup_hint_line_for_keymap;
use crate::chatwidget::ThreadInputStateRestoreMode;
+use crate::chatwidget::UserMessage;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SessionSource;
use codex_app_server_protocol::Thread;
@@ -20,7 +21,10 @@ use codex_app_server_protocol::ThreadHistoryMode;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
use codex_app_server_protocol::UserInput;
+use codex_protocol::models::snapshot_local_user_input;
+use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::SubAgentSource;
+use codex_protocol::user_input::UserInput as CoreUserInput;
pub(crate) const AGENTS_OVERVIEW_VIEW_ID: &str = "agents-overview";
@@ -615,7 +619,7 @@ impl App {
pub(super) async fn dispatch_agents_overview_task(
&mut self,
app_server: &mut AppServerSession,
- prompt: String,
+ prompt: UserMessage,
cwd: Option<AbsolutePathBuf>,
) {
self.refresh_in_memory_config_from_disk_best_effort("starting a background task")
@@ -720,6 +724,69 @@ impl App {
.or_else(|| self.model_catalog.models.first())
.map(|model| model.model.clone());
}
+ let model = config.model.as_deref().or_else(|| {
+ self.model_catalog
+ .models
+ .iter()
+ .find(|model| model.is_default)
+ .or_else(|| self.model_catalog.models.first())
+ .map(|model| model.model.as_str())
+ });
+ if !prompt.local_images.is_empty()
+ && let Some(model) = model
+ && self.model_catalog.models.iter().any(|preset| {
+ preset.model == model && !preset.input_modalities.contains(&InputModality::Image)
+ })
+ {
+ let message = format!(
+ "Model {model} does not support image inputs. Remove images or switch models."
+ );
+ self.restore_agents_overview_prompt(prompt);
+ self.chat_widget.add_error_message(message);
+ return;
+ }
+ let images = prompt
+ .local_images
+ .iter()
+ .map(|image| {
+ Ok(CoreUserInput::LocalImage {
+ path: std::path::absolute(&image.path)?,
+ detail: None,
+ })
+ })
+ .collect::<std::io::Result<Vec<_>>>();
+ let images = match images {
+ Ok(images) => images,
+ Err(error) => {
+ self.restore_agents_overview_prompt(prompt);
+ self.chat_widget
+ .add_error_message(format!("Failed to prepare image: {error}"));
+ return;
+ }
+ };
+ let images = if app_server.uses_remote_workspace() && !images.is_empty() {
+ match tokio::task::spawn_blocking(move || {
+ let mut images = images;
+ for image in &mut images {
+ snapshot_local_user_input(image)?;
+ }
+ Ok::<_, std::io::Error>(images)
+ })
+ .await
+ .map_err(std::io::Error::other)
+ .and_then(|result| result)
+ {
+ Ok(images) => images,
+ Err(error) => {
+ self.restore_agents_overview_prompt(prompt);
+ self.chat_widget
+ .add_error_message(format!("Failed to prepare image: {error}"));
+ return;
+ }
+ }
+ } else {
+ images
+ };
match app_server
.start_thread_with_session_start_source(
&self.local_settings,
@@ -735,8 +802,13 @@ impl App {
self.agents_overview
.dispatched_requests
.insert(thread_id, Vec::new());
- self.submit_agents_overview_prompt(app_server, thread_id, prompt)
- .await;
+ self.submit_agents_overview_prompt(
+ app_server,
+ thread_id,
+ prompt,
+ images.into_iter().map(Into::into).collect(),
+ )
+ .await;
}
Err(error) => {
self.restore_agents_overview_prompt(prompt);
@@ -750,18 +822,27 @@ impl App {
&mut self,
app_server: &AppServerSession,
thread_id: ThreadId,
- prompt: String,
+ prompt: UserMessage,
+ mut input: Vec<UserInput>,
) {
+ if !prompt.text.is_empty() {
+ input.push(UserInput::Text {
+ text: prompt.text.clone(),
+ text_elements: prompt
+ .text_elements
+ .iter()
+ .cloned()
+ .map(Into::into)
+ .collect(),
+ });
+ }
let result = app_server
.request_handle()
.request_typed::<TurnStartResponse>(ClientRequest::TurnStart {
request_id: RequestId::String(Uuid::new_v4().to_string()),
params: TurnStartParams {
thread_id: thread_id.to_string(),
- input: vec![UserInput::Text {
- text: prompt.clone(),
- text_elements: Vec::new(),
- }],
+ input,
..Default::default()
},
})
diff --git a/codex-rs/tui/src/app/agents_overview_composer.rs b/codex-rs/tui/src/app/agents_overview_composer.rs
index 7a15da32f0..5d482a9ea4 100644
--- a/codex-rs/tui/src/app/agents_overview_composer.rs
+++ b/codex-rs/tui/src/app/agents_overview_composer.rs
@@ -4,6 +4,7 @@
use super::*;
use crate::bottom_pane::ChatComposer;
use crate::bottom_pane::ChatComposerConfig;
+use crate::chatwidget::UserMessage;
impl App {
pub(super) fn sync_agents_overview_composer(&self) {
@@ -21,6 +22,7 @@ impl App {
self.config.disable_paste_burst,
ChatComposerConfig {
trim_submission: false,
+ image_paste_enabled: true,
..ChatComposerConfig::plain_text()
},
);
@@ -39,18 +41,37 @@ impl App {
composer.set_keymap_bindings(&self.keymap);
}
- pub(super) fn restore_agents_overview_prompt(&mut self, prompt: String) {
+ pub(super) fn restore_agents_overview_prompt(&mut self, prompt: UserMessage) {
if let Ok(mut state) = self.agents_overview.view_state.lock()
&& let Some(composer) = state.composer.as_mut()
&& composer.is_empty()
&& !composer.is_in_paste_burst()
&& !composer.popup_active()
{
- composer.set_text_content(prompt, Vec::new(), Vec::new());
+ composer.set_text_content(
+ prompt.text,
+ prompt.text_elements,
+ prompt
+ .local_images
+ .into_iter()
+ .map(|image| image.path)
+ .collect(),
+ );
composer.move_cursor_to_end();
return;
}
+ let hint = (!prompt.local_images.is_empty()).then(|| {
+ format!(
+ "Reattach images from: {}",
+ prompt
+ .local_images
+ .iter()
+ .map(|image| image.path.display().to_string())
+ .collect::<Vec<_>>()
+ .join(", ")
+ )
+ });
self.chat_widget
- .add_info_message(format!("Unsent task: {prompt}"), /*hint*/ None);
+ .add_info_message(format!("Unsent task: {}", prompt.text), hint);
}
}
diff --git a/codex-rs/tui/src/app/agents_overview_input.rs b/codex-rs/tui/src/app/agents_overview_input.rs
index 9be8c33766..c2a194a4dd 100644
--- a/codex-rs/tui/src/app/agents_overview_input.rs
+++ b/codex-rs/tui/src/app/agents_overview_input.rs
@@ -3,6 +3,10 @@
use super::*;
use crate::bottom_pane::InputResult;
+use crate::chatwidget::UserMessage;
+use crate::clipboard_paste::paste_image_to_temp_png;
+use crossterm::event::KeyEventKind;
+use crossterm::event::KeyModifiers;
impl AgentsOverviewView {
pub(super) fn handle_composer_key(&mut self, key: KeyEvent) {
@@ -16,6 +20,20 @@ impl AgentsOverviewView {
let Some(composer) = state.composer.as_mut() else {
return;
};
+ if key.kind == KeyEventKind::Press
+ && matches!(key.code, KeyCode::Char('v' | 'V'))
+ && key
+ .modifiers
+ .intersects(KeyModifiers::CONTROL | KeyModifiers::ALT)
+ {
+ match paste_image_to_temp_png() {
+ Ok((path, _)) => composer.attach_image(path),
+ Err(error) => self.app_event_tx.send(AppEvent::InsertHistoryCell(Box::new(
+ crate::history_cell::new_error_event(format!("Failed to paste image: {error}")),
+ ))),
+ }
+ return;
+ }
if offline
&& !composer.popup_active()
&& (self.composer_keymap.submit.is_pressed(key)
@@ -27,11 +45,26 @@ impl AgentsOverviewView {
return;
}
let (result, _) = composer.handle_key_event(key);
+ let prompt = if let InputResult::Submitted {
+ text,
+ text_elements,
+ } = result
+ {
+ Some(UserMessage {
+ text,
+ text_elements,
+ local_images: composer.take_recent_submission_images_with_placeholders(),
+ remote_image_urls: Vec::new(),
+ mention_bindings: Vec::new(),
+ })
+ } else {
+ None
+ };
drop(state);
- if let InputResult::Submitted { text, .. } = result {
+ if let Some(prompt) = prompt {
self.app_event_tx
.send(AppEvent::DispatchAgentsOverviewTask {
- prompt: text,
+ prompt,
cwd: (!status_grouping)
.then(|| self.selected_row().map(|row| row.thread.cwd.clone()))
.flatten(),
diff --git a/codex-rs/tui/src/app/agents_overview_tests.rs b/codex-rs/tui/src/app/agents_overview_tests.rs
index c7e9f4def2..6dfe993cb7 100644
--- a/codex-rs/tui/src/app/agents_overview_tests.rs
+++ b/codex-rs/tui/src/app/agents_overview_tests.rs
@@ -145,7 +145,7 @@ async fn overview_composer_preserves_editing_and_routes_focus() {
view.handle_key_event(KeyCode::Esc.into());
view.handle_key_event(KeyCode::Tab.into());
assert!(
- matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt == expected.0)
+ matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt.text == expected.0)
);
view.handle_key_event(KeyCode::Up.into());
assert_eq!(overview_draft(&app).0, expected.0);
@@ -161,7 +161,7 @@ async fn overview_composer_preserves_editing_and_routes_focus() {
app.dispatch_agents_overview_task(&mut server, "retry me".into(), Some(app.config.cwd.clone()))
.await;
view.handle_paste(" later".into());
- app.submit_agents_overview_prompt(&server, thread_id, "older failure".into())
+ app.submit_agents_overview_prompt(&server, thread_id, "older failure".into(), Vec::new())
.await;
assert_eq!(overview_draft(&app).0, "retry me later");
server.shutdown().await.unwrap();
@@ -222,7 +222,7 @@ async fn overview_composer_preserves_pastes_and_editor_bindings() {
view.handle_key_event(KeyCode::Char('x').into());
view.handle_key_event(KeyCode::F(8).into());
assert!(
- matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt == format!("{pasted}\nabcxlast line"))
+ matches!(rx.try_recv(), Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. }) if prompt.text == format!("{pasted}\nabcxlast line"))
);
}
@@ -1094,7 +1094,7 @@ async fn shared_overview_shows_only_root_sessions() {
assert!(matches!(
event_rx.try_recv(),
Ok(AppEvent::DispatchAgentsOverviewTask { prompt, cwd: None })
- if prompt == "Use the current project"
+ if prompt.text == "Use the current project"
));
action_view.handle_key_event(KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE));
action_view.handle_key_event(KeyEvent::new(KeyCode::Char('s'), KeyModifiers::CONTROL));
@@ -1114,7 +1114,7 @@ async fn shared_overview_shows_only_root_sessions() {
assert!(matches!(
event_rx.try_recv(),
Ok(AppEvent::DispatchAgentsOverviewTask { prompt, cwd: Some(cwd) })
- if prompt == "Fix the flaky tests after all retries complete"
+ if prompt.text == "Fix the flaky tests after all retries complete"
&& cwd == test_path_buf("/tmp/project").abs()
));
assert!(action_view.handle_paste(" ".to_string()));
@@ -1131,7 +1131,7 @@ async fn shared_overview_shows_only_root_sessions() {
assert!(matches!(
event_rx.try_recv(),
Ok(AppEvent::DispatchAgentsOverviewTask { prompt, .. })
- if prompt.trim() == "Continue working"
+ if prompt.text.trim() == "Continue working"
));
action_view.handle_key_event(KeyEvent::new(KeyCode::Char('n'), KeyModifiers::CONTROL));
action_view.handle_paste("First line\nSecond line".into());
diff --git a/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs b/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs
index 504f0f7c84..d3a9f24fa2 100644
--- a/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs
+++ b/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs
@@ -5,10 +5,220 @@ use crate::app::agents_overview::AGENTS_OVERVIEW_VIEW_ID;
use crate::app::tests::session_lifecycle_requests::HistoryCapabilities;
use crate::app::tests::session_lifecycle_requests::recorded_params;
use crate::app::tests::session_lifecycle_requests::start_recording_app_server_with_history;
+use crate::bottom_pane::BottomPaneView;
+use crate::bottom_pane::LocalImageAttachment;
+use crate::chatwidget::UserMessage;
+use crate::chatwidget::tests::helpers::render_bottom_popup;
+use crate::model_catalog::ModelCatalog;
use crate::test_support::PathBufExt;
+use codex_app_server_protocol::UserInput;
+use codex_protocol::openai_models::InputModality;
use codex_state::SqliteConfig;
+use crossterm::event::KeyCode;
use pretty_assertions::assert_eq;
+#[tokio::test]
+async fn background_task_sends_pasted_image_with_first_prompt() -> Result<()> {
+ let (mut app, mut events, _) = make_test_app_with_channels().await;
+ let image_dir = tempdir()?;
+ let image_path = image_dir.path().join("pasted.png");
+ image::RgbImage::new(1, 1).save(&image_path)?;
+ let pasted_path = pathdiff::diff_paths(&image_path, std::env::current_dir()?)
+ .unwrap_or_else(|| image_path.clone());
+ let mut view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);
+ assert!(view.handle_paste(pasted_path.to_string_lossy().into_owned()));
+ assert!(view.handle_paste("Describe this".into()));
+ let rendered_view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);
+ app.chat_widget
+ .show_bottom_pane_view(Box::new(rendered_view));
+ insta::assert_snapshot!(
+ render_bottom_popup(&app.chat_widget, /*width*/ 80)
+ .lines()
+ .find(|line| line.contains("[Image #1]"))
+ .expect("image attachment visible"),
+ @"› [Image #1] Describe this"
+ );
+ view.handle_key_event(KeyCode::Enter.into());
+ let prompt = match events.try_recv()? {
+ AppEvent::DispatchAgentsOverviewTask { prompt, .. } => prompt,
+ event => panic!("expected task dispatch, got {event:?}"),
+ };
+ assert_eq!(prompt.local_images[0].path, pasted_path);
+ assert_eq!(
+ prompt.text_elements[0].placeholder(&prompt.text),
+ Some("[Image #1]")
+ );
+
+ let (mut server, requests, proxy) = start_recording_app_server_with_history(
+ &app.config,
+ HistoryCapabilities::Current,
+ /*blocked_thread_list*/ None,
+ /*failed_thread_name*/ None,
+ crate::app_server_session::ThreadParamsMode::Embedded,
+ LoaderOverrides::default(),
+ )
+ .await?;
+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), /*cwd*/ None)
+ .await;
+ let turns = recorded_params(&requests, "turn/start");
+ assert_eq!(turns.len(), 1);
+ assert_eq!(
+ turns[0]["input"],
+ serde_json::to_value(vec![
+ UserInput::LocalImage {
+ path: std::path::absolute(&pasted_path)?,
+ detail: None
+ },
+ UserInput::Text {
+ text: prompt.text.clone(),
+ text_elements: prompt
+ .text_elements
+ .iter()
+ .cloned()
+ .map(Into::into)
+ .collect(),
+ },
+ ])?
+ );
+
+ app.cli_kv_overrides = vec![("model".into(), toml::Value::Integer(1))];
+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), Some(app.config.cwd.clone()))
+ .await;
+ let restored = app
+ .agents_overview
+ .view_state
+ .lock()
+ .unwrap()
+ .composer
+ .as_ref()
+ .unwrap()
+ .draft_snapshot();
+ assert_eq!(
+ (restored.text, restored.text_elements, restored.local_images),
+ (
+ prompt.text.clone(),
+ prompt.text_elements.clone(),
+ prompt.local_images.clone()
+ )
+ );
+ view.handle_key_event(KeyCode::Enter.into());
+ assert!(view.handle_paste("A newer draft".into()));
+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), Some(app.config.cwd.clone()))
+ .await;
+ let current = app
+ .agents_overview
+ .view_state
+ .lock()
+ .unwrap()
+ .composer
+ .as_ref()
+ .unwrap()
+ .draft_snapshot();
+ assert_eq!(current.text, "A newer draft");
+ let notice = std::iter::from_fn(|| events.try_recv().ok())
+ .filter_map(|event| match event {
+ AppEvent::InsertHistoryCell(cell) => {
+ Some(lines_to_single_string(&cell.display_lines(/*width*/ 200)))
+ }
+ _ => None,
+ })
+ .find(|message| message.contains("Reattach images from:"))
+ .expect("attachment recovery notice");
+ assert!(notice.contains(&pasted_path.display().to_string()));
+ server.shutdown().await?;
+ proxy.await??;
+ Ok(())
+}
+
+#[tokio::test]
+async fn remote_background_task_sends_clipboard_image_bytes() -> Result<()> {
+ let mut app = make_test_app_with_channels().await.0;
+ let image_dir = tempdir()?;
+ let image_path = image_dir.path().join("pasted.png");
+ image::RgbImage::new(1, 1).save(&image_path)?;
+ let mut prompt = UserMessage::from("Describe this image");
+ prompt.local_images.push(LocalImageAttachment {
+ placeholder: "[Image #1]".into(),
+ path: image_path,
+ });
+ let (mut server, requests, proxy) = start_recording_app_server_with_history(
+ &app.config,
+ HistoryCapabilities::Current,
+ /*blocked_thread_list*/ None,
+ /*failed_thread_name*/ None,
+ crate::app_server_session::ThreadParamsMode::Remote,
+ LoaderOverrides::default(),
+ )
+ .await?;
+ app.dispatch_agents_overview_task(&mut server, prompt, /*cwd*/ None)
+ .await;
+ let turns = recorded_params(&requests, "turn/start");
+ assert_eq!(turns.len(), 1);
+ let image = &turns[0]["input"][0];
+ assert_eq!(image["type"], "image");
+ assert!(
+ image["url"]
+ .as_str()
+ .is_some_and(|url| url.starts_with("data:image/png;base64,"))
+ );
+ assert_eq!(turns[0]["input"][1]["text"], "Describe this image");
+ server.shutdown().await?;
+ proxy.await??;
+ Ok(())
+}
+
+#[tokio::test]
+async fn background_task_rejects_images_for_text_only_model() -> Result<()> {
+ let (mut app, mut events, _) = make_test_app_with_channels().await;
+ let mut preset = app.model_catalog.models[0].clone();
+ preset.model = "text-only-test-model".into();
+ preset.input_modalities = vec![InputModality::Text];
+ app.model_catalog = std::sync::Arc::new(ModelCatalog::new(vec![preset]));
+ app.harness_overrides.model = Some("text-only-test-model".into());
+ let view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);
+ app.chat_widget.show_bottom_pane_view(Box::new(view));
+ let mut prompt = UserMessage::from("[Image #1] Describe this");
+ prompt.local_images.push(LocalImageAttachment {
+ placeholder: "[Image #1]".into(),
+ path: "/tmp/test-pasted.png".into(),
+ });
+ let (mut server, requests, proxy) = start_recording_app_server_with_history(
+ &app.config,
+ HistoryCapabilities::Current,
+ /*blocked_thread_list*/ None,
+ /*failed_thread_name*/ None,
+ crate::app_server_session::ThreadParamsMode::Embedded,
+ LoaderOverrides::default(),
+ )
+ .await?;
+ app.dispatch_agents_overview_task(&mut server, prompt.clone(), /*cwd*/ None)
+ .await;
+ assert!(recorded_params(&requests, "thread/start").is_empty());
+ let draft = app
+ .agents_overview
+ .view_state
+ .lock()
+ .unwrap()
+ .composer
+ .as_ref()
+ .unwrap()
+ .draft_snapshot();
+ assert_eq!(draft.local_images, prompt.local_images);
+ let error = std::iter::from_fn(|| events.try_recv().ok())
+ .filter_map(|event| match event {
+ AppEvent::InsertHistoryCell(cell) => {
+ Some(lines_to_single_string(&cell.display_lines(/*width*/ 100)))
+ }
+ _ => None,
+ })
+ .find(|message| message.contains("does not support image inputs"))
+ .expect("visible model rejection");
+ insta::assert_snapshot!(error, @"■ Model text-only-test-model does not support image inputs. Remove images or switch models.");
+ server.shutdown().await?;
+ proxy.await??;
+ Ok(())
+}
+
#[tokio::test]
async fn background_task_reads_server_defaults_for_actual_destination() -> Result<()> {
for (mode, explicit_cwd, launch_override, expected_cwd, expected_model) in [
diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs
index 9ad0b0b034..c397c98f64 100644
--- a/codex-rs/tui/src/app_event.rs
+++ b/codex-rs/tui/src/app_event.rs
@@ -278,7 +278,7 @@ pub(crate) enum AppEvent {
},
/// Start a background task directly from the shared dashboard.
DispatchAgentsOverviewTask {
- prompt: String,
+ prompt: UserMessage,
cwd: Option<AbsolutePathBuf>,
},
/// Rename a task directly from the shared dashboard.