Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

Agent chat: keep Send in view for a long message, open "View attached" (#2224) (#2225)

Rec0iLf498d23a79ec
diff --git a/crates/ui-egui/src/agent_chat/mod.rs b/crates/ui-egui/src/agent_chat/mod.rs
index 52276420fb..88d62f820b 100644
--- a/crates/ui-egui/src/agent_chat/mod.rs
+++ b/crates/ui-egui/src/agent_chat/mod.rs
@@ -56,6 +56,8 @@ const FILE_NAMES_ROOM: usize = 2_000;
const APPROVAL_TIMEOUT_MS: f64 = 240_000.0;
/// Size of the picture of the view sent with a message.
const VIEW_IMAGE: (u32, u32) = (960, 600);
+/// A picture kept for the user to open later, at most this big.
+const MAX_PICTURE_BYTES: usize = 4 << 20;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum State {
@@ -93,11 +95,58 @@ impl State {
}
}
+/// A picture shown with a message: what the agent got (a picture of the view, or any other
+/// attachment) as PNG, and a note shown with it in the viewer. The thumbnail and the viewer share
+/// one texture, made when the picture is first drawn.
+#[derive(Clone)]
+pub struct Picture {
+ png: Arc<Vec<u8>>,
+ note: String,
+ texture: Arc<std::sync::OnceLock<Option<egui::TextureHandle>>>,
+}
+
+impl Picture {
+ pub fn new(png: Vec<u8>, note: impl Into<String>) -> Self {
+ Self { png: Arc::new(png), note: note.into(), texture: Arc::default() }
+ }
+
+ /// The picture as a texture, `None` when it can't be decoded.
+ #[cfg(not(target_arch = "wasm32"))]
+ fn texture(&self, ctx: &egui::Context) -> Option<egui::TextureHandle> {
+ self.texture
+ .get_or_init(|| {
+ let rgba = image::load_from_memory_with_format(&self.png, image::ImageFormat::Png).ok()?.to_rgba8();
+ let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?];
+ Some(ctx.load_texture(
+ "sc_agent_chat_picture",
+ egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()),
+ egui::TextureOptions::LINEAR,
+ ))
+ })
+ .clone()
+ }
+}
+
+impl std::fmt::Debug for Picture {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "Picture({} bytes, {:?})", self.png.len(), self.note)
+ }
+}
+
+/// The viewer: the pictures of one message, and the one shown.
+#[derive(Clone, Debug)]
+pub struct Gallery {
+ pictures: Vec<Picture>,
+ at: usize,
+}
+
#[derive(Clone, Debug)]
pub enum Item {
User {
text: String,
image: bool,
+ /// The view's picture, kept once it is taken (when the agent starts on the message).
+ pictures: Vec<Picture>,
/// The names of the files that went with it.
files: Vec<String>,
},
@@ -202,6 +251,8 @@ pub struct AgentChat {
pub open: bool,
pub items: Vec<Item>,
pub input: String,
+ /// The viewer, open on one message's pictures.
+ gallery: Option<Gallery>,
state: Option<State>,
agent: String,
images: bool,
@@ -209,8 +260,9 @@ pub struct AgentChat {
pub attach_view: bool,
auth: Vec<acp::AuthMethod>,
auth_status: Option<String>,
- /// A message typed before the agent was ready, with its files.
- queued: Option<(String, bool, Vec<attach::Attachment>)>,
+ /// A message typed before the agent was ready: its text, whether the view goes with it, its
+ /// place in the conversation, and its files.
+ queued: Option<(String, bool, Option<usize>, Vec<attach::Attachment>)>,
/// Messages typed while the agent works, in order. They go as one prompt when the turn ends
/// (or at once with Send now); after Stop they wait for Send.
pub pending: Vec<Pending>,
@@ -231,7 +283,7 @@ pub struct AgentChat {
/// Where the panel was drawn last (for its shortcuts).
pub panel_rect: Option<egui::Rect>,
/// The last message sent, kept to send again after a sign-in (its files too).
- last: Option<(String, bool, Vec<attach::Attachment>)>,
+ last: Option<(String, bool, Option<usize>, Vec<attach::Attachment>)>,
/// Files for the next message (pictures, text files, imported parts), shown as chips.
pub(crate) attachments: Vec<attach::Attachment>,
/// Pictures pasted so far, to name the next one.
@@ -347,6 +399,17 @@ impl AgentChat {
}
self.items.push(item);
}
+ /// Keep the picture just sent with the message at `at` in the conversation (the user can open
+ /// it). A message sent again (after a sign-in) has its picture already: nothing is added, and
+ /// never to another message.
+ fn keep_picture(&mut self, at: Option<usize>, png: Vec<u8>) {
+ if let Some(Item::User { image: true, pictures, .. }) = at.and_then(|i| self.items.get_mut(i))
+ && pictures.is_empty()
+ && png.len() <= MAX_PICTURE_BYTES
+ {
+ pictures.push(Picture::new(png, ""));
+ }
+ }
fn notice(&mut self, text: impl Into<String>, error: bool) {
self.push(Item::Notice { text: text.into(), error });
}
@@ -704,11 +767,13 @@ pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) {
/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it
/// cut in on its previous step).
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec<attach::Attachment>, interrupted: bool) {
- app.agent_chat.push(Item::User { text: text.clone(), image, files: files.iter().map(|f| f.name.clone()).collect() });
+ app.agent_chat.push(Item::User { text: text.clone(), image, pictures: Vec::new(), files: files.iter().map(|f| f.name.clone()).collect() });
+ // The message's place in the conversation: its picture is kept there when it is taken.
+ let at = app.agent_chat.items.len().checked_sub(1);
match app.agent_chat.state() {
- State::Ready => start_turn_with(app, text, image, files, interrupted.then_some(INTERRUPT_NOTE)),
+ State::Ready => start_turn_with(app, text, image, at, files, interrupted.then_some(INTERRUPT_NOTE)),
s => {
- app.agent_chat.queued = Some((text, image, files));
+ app.agent_chat.queued = Some((text, image, at, files));
#[cfg(not(target_arch = "wasm32"))]
if !s.running() {
connect(app, ctx);
@@ -775,6 +840,7 @@ pub fn new_chat(app: &mut SolveApp) {
let chat = &mut app.agent_chat;
chat.confirm_new = false;
chat.items.clear();
+ chat.gallery = None;
chat.pending.clear();
chat.attachments.clear();
chat.loading.clear();
@@ -823,14 +889,14 @@ fn after_turn(app: &mut SolveApp, ctx: &egui::Context) {
}
}
-fn start_turn(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>) {
- start_turn_with(app, text, image, files, None);
+fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>) {
+ start_turn_with(app, text, image, at, files, None);
}
/// Start a message; `note` goes before the text for the agent, not into the conversation. The
/// files are named after the text (text files in full), and pictures go as images to an agent
-/// that takes them.
-fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>, note: Option<&str>) {
+/// that takes them. `at` is the message's place in the conversation, where its picture is kept.
+fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>, note: Option<&str>) {
let pictures = app.agent_chat.images;
let lead = match note {
Some(n) => format!("{n}\n\n{text}"),
@@ -843,7 +909,10 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<att
let mut parts = vec![PromptPart::Text(sent)];
if image {
match crate::control::view_png(app, VIEW_IMAGE.0, VIEW_IMAGE.1) {
- Ok(png) => parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) }),
+ Ok(png) => {
+ parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) });
+ app.agent_chat.keep_picture(at, png);
+ }
Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"),
}
}
@@ -855,7 +924,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<att
let label: String = first_line.chars().take(48).collect();
app.agent_chat.turn =
Some(Turn { start_doc: app.session.doc.clone(), start_undo: app.session.undo.len(), changes: 0, requests: 0, limit_told: false, label });
- app.agent_chat.last = Some((text, image, files));
+ app.agent_chat.last = Some((text, image, at, files));
app.agent_chat.state = Some(State::Busy);
app.agent_chat.stopped = false;
app.agent_chat.steering = false;
@@ -1258,8 +1327,8 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
if ready {
apply_saved_model(app);
}
- if ready && let Some((text, image, files)) = app.agent_chat.queued.take() {
- start_turn(app, text, image, files);
+ if ready && let Some((text, image, at, files)) = app.agent_chat.queued.take() {
+ start_turn(app, text, image, at, files);
}
}
pump_tools(app);
@@ -1478,7 +1547,7 @@ pub fn status(app: &SolveApp) -> Value {
.items
.iter()
.map(|i| match i {
- Item::User { text, image, files } => json!({"user": text, "image": image, "files": files}),
+ Item::User { text, image, pictures, files } => json!({"user": text, "image": image, "pictures": pictures.len(), "files": files}),
Item::Agent { thought, text } => json!({"agent": text, "thought": thought}),
Item::Tool { id, title, kind, status, output } => {
json!({"tool": id, "title": title, "kind": kind, "status": status.map(|s| format!("{s:?}")), "output": output})
diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rs
index 0b75e283d4..66358816c6 100644
--- a/crates/ui-egui/src/agent_chat/panel.rs
+++ b/crates/ui-egui/src/agent_chat/panel.rs
@@ -18,6 +18,12 @@ fn panel_id(left: bool) -> Id {
Id::new(if left { "sc_agent_chat_left" } else { "sc_agent_chat_right" })
}
+/// The largest thumbnail of a message's picture, and its largest preview while hovered (points).
+#[cfg(not(target_arch = "wasm32"))]
+const THUMB_MAX: (f32, f32) = (150.0, 56.0);
+#[cfg(not(target_arch = "wasm32"))]
+const PREVIEW_MAX: (f32, f32) = (560.0, 380.0);
+
/// Forget the panel's own idea of its width, so the preference applies on the next frame.
pub fn forget_width(ctx: &egui::Context) {
ctx.data_mut(|d| {
@@ -105,6 +111,8 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
.show(ui, |ui| body(app, ui, left));
crate::scenario::publish_rect("agent_chat:panel", r.response.rect);
app.agent_chat.panel_rect = Some(r.response.rect);
+ #[cfg(not(target_arch = "wasm32"))]
+ gallery_window(app, &ctx);
app.agent_chat.drop_rect = Some(r.response.rect);
show_drop_hint(app, &ctx);
let w = r.response.rect.width();
@@ -113,6 +121,88 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
}
}
+/// The pictures of a message as small thumbnails in a row. Hovering one shows it bigger for as long
+/// as the pointer stays; a click returns its place in the row, to open the viewer there.
+#[cfg(not(target_arch = "wasm32"))]
+fn picture_strip(ui: &mut egui::Ui, n: usize, pictures: &[Picture]) -> Option<usize> {
+ let t = Tokens::get();
+ let mut clicked = None;
+ ui.horizontal_wrapped(|ui| {
+ for (i, picture) in pictures.iter().enumerate() {
+ let Some(texture) = picture.texture(ui.ctx()) else {
+ ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
+ continue;
+ };
+ let size = fit(texture.size_vec2(), vec2(THUMB_MAX.0, THUMB_MAX.1));
+ let thumb = ui.add(egui::Image::new((texture.id(), size)).corner_radius(4.0).sense(Sense::click()));
+ crate::scenario::publish_handle(&format!("agent_chat:picture:{n}:{i}"), thumb.rect.center());
+ let hover_max = (ui.ctx().content_rect().size() * 0.5).min(vec2(PREVIEW_MAX.0, PREVIEW_MAX.1));
+ let thumb = thumb.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_ui(|ui| {
+ ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), hover_max))).corner_radius(4.0));
+ if !picture.note.is_empty() {
+ ui.label(&picture.note);
+ }
+ });
+ if thumb.clicked() {
+ clicked = Some(i);
+ }
+ }
+ });
+ clicked
+}
+
+/// `size` scaled to fit in `max`, keeping its shape (never below one point).
+#[cfg(not(target_arch = "wasm32"))]
+fn fit(size: egui::Vec2, max: egui::Vec2) -> egui::Vec2 {
+ let scale = (max.x / size.x.max(1.0)).min(max.y / size.y.max(1.0)).min(2.0);
+ vec2((size.x * scale).max(1.0), (size.y * scale).max(1.0))
+}
+
+/// The viewer: a message's pictures over the app, one at a time with a counter, Previous and Next
+/// (also the arrow keys), closed with the button, Esc or a click outside.
+#[cfg(not(target_arch = "wasm32"))]
+fn gallery_window(app: &mut SolveApp, ctx: &egui::Context) {
+ let Some(mut gallery) = app.agent_chat.gallery.take() else { return };
+ let count = gallery.pictures.len();
+ gallery.at = gallery.at.min(count.saturating_sub(1));
+ let (left, right) =
+ ctx.input_mut(|i| (i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowLeft), i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowRight)));
+ let mut step: isize = if left { -1 } else { isize::from(right) };
+ let t = Tokens::get();
+ let max = ctx.content_rect().size() * vec2(0.8, 0.7);
+ let modal = egui::Modal::new(Id::new("sc_agent_chat_gallery")).show(ctx, |ui| {
+ ui.set_max_width(max.x);
+ ui.horizontal(|ui| {
+ let prev = ui.add_enabled(count > 1, egui::Button::new("‹"));
+ crate::scenario::publish_handle("agent_chat:gallery_prev", prev.rect.center());
+ let next = ui.add_enabled(count > 1, egui::Button::new("›"));
+ crate::scenario::publish_handle("agent_chat:gallery_next", next.rect.center());
+ if prev.clicked() {
+ step = -1;
+ }
+ if next.clicked() {
+ step = 1;
+ }
+ ui.label(RichText::new(format!("{} / {}", gallery.at + 1, count)).color(t.text_dim));
+ });
+ if let Some(picture) = gallery.pictures.get(gallery.at) {
+ if let Some(texture) = picture.texture(ui.ctx()) {
+ let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0));
+ crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect);
+ }
+ if !picture.note.is_empty() {
+ ui.add(egui::Label::new(&picture.note).wrap());
+ }
+ }
+ });
+ if count > 0 && step != 0 {
+ gallery.at = (gallery.at as isize + step).rem_euclid(count as isize) as usize;
+ }
+ if !modal.should_close() {
+ app.agent_chat.gallery = Some(gallery);
+ }
+}
+
/// While files are dragged over the window: if they would drop on the chat, say so over it.
fn show_drop_hint(app: &SolveApp, ctx: &egui::Context) {
#[cfg(not(target_arch = "wasm32"))]
@@ -193,6 +283,8 @@ fn window(app: &mut SolveApp, ctx: &egui::Context) {
let t = Tokens::get();
let frame = egui::Frame::NONE.fill(t.panel).inner_margin(egui::Margin::symmetric(8, 6));
let r = egui::CentralPanel::default().frame(frame).show(ui, |ui| body(app, ui, false));
+ // The picture viewer opens over this window, where the click was, not over the main one.
+ gallery_window(app, ui.ctx());
if class == egui::ViewportClass::EmbeddedWindow {
// Floating inside the app: its handles are the main window's, as the panel's are.
crate::scenario::publish_rect("agent_chat:window", r.response.rect);
@@ -468,21 +560,12 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
auth_box(app, ui);
}
ui.separator();
- // The input sits at the bottom (with a row per line of file chips); the conversation takes
- // the rest.
- let input_h = 92.0 + chip_rows(app.agent_chat.files(), ui.available_width(), ui.spacing().item_spacing.x) as f32 * 32.0;
- let avail = (ui.available_height() - input_h).max(80.0);
- egui::ScrollArea::vertical().id_salt("sc_agent_chat_scroll").max_height(avail).auto_shrink([false, false]).stick_to_bottom(true).show(ui, |ui| {
- conversation(app, ui);
- });
- ui.separator();
+ // The input sits at the bottom of the panel and grows upwards: a long message makes its box
+ // taller up to `edit_max`, then it scrolls, and the conversation takes what is left above. It
+ // is a panel of its own at the bottom, as high as its content this frame, so the buttons under
+ // the box are always in view.
+ let edit_max = (ui.available_height() * 0.4).clamp(48.0, 160.0);
let busy = state == State::Busy;
- attachment_chips(app, ui);
- let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
- .hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
- .desired_rows(2)
- .desired_width(f32::INFINITY)
- .id_salt("sc_agent_chat_input");
// Enter sends (queues while the agent works); Ctrl/Cmd+Enter sends now; Esc stops. They are
// read before the field sees them so no newline is typed.
let focused = ui.memory(|m| m.has_focus(Id::new("sc_agent_chat_input")));
@@ -491,52 +574,77 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
} else {
(false, false)
};
- let r = ui.add(edit);
- crate::scenario::publish_rect("agent_chat:input", r.rect);
- // macOS shows a window's cursors only while it is the active window: over the text box of a
- // chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat
- // window becomes active, so the text cursor shows and the first click places the caret.
- let (chat_active, main_active) = (ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused));
- if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) {
- ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus);
- }
- let enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift);
- if esc {
+ let (mut send_clicked, mut stop_clicked, mut enter) = (false, false, false);
+ egui::Panel::bottom(Id::new("sc_agent_chat_composer")).resizable(false).show_separator_line(false).frame(egui::Frame::NONE).show(ui, |ui| {
+ ui.separator();
+ // The files for the next message (#1744), as chips of their own.
+ attachment_chips(app, ui);
+ let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
+ .hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
+ .desired_rows(2)
+ .desired_width(f32::INFINITY)
+ .id_salt("sc_agent_chat_input");
+ let r = egui::ScrollArea::vertical()
+ .id_salt("sc_agent_chat_input_scroll")
+ .max_height(edit_max)
+ .auto_shrink([false, true])
+ .show(ui, |ui| ui.add(edit))
+ .inner;
+ crate::scenario::publish_rect("agent_chat:input", r.rect);
+ // macOS shows a window's cursors only while it is the active window: over the text box of a
+ // chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat
+ // window becomes active, so the text cursor shows and the first click places the caret.
+ let (chat_active, main_active) =
+ (ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused));
+ if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) {
+ ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus);
+ }
+ enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift);
+ ui.horizontal(|ui| {
+ add_menu(app, ui);
+ // A message waits for files still being read.
+ let reading = !app.agent_chat.loading.is_empty();
+ let send_b = ui
+ .add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
+ .on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
+ crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
+ send_clicked = send_b.clicked();
+ stop_clicked = busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked();
+ // A picture of the view goes with the message to agents that take pictures.
+ let images = app.agent_chat.images || !state.running();
+ let attach = ui
+ .add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view")))
+ .on_hover_text(tr("ui.agent_chat_attach_view_tip"))
+ .on_disabled_hover_text(tr("ui.agent_chat_no_images"));
+ crate::scenario::publish_handle("agent_chat:attach", attach.rect.center());
+ });
+ });
+ egui::ScrollArea::vertical()
+ .id_salt("sc_agent_chat_scroll")
+ .max_height(ui.available_height())
+ .auto_shrink([false, false])
+ .stick_to_bottom(true)
+ .show(ui, |ui| {
+ conversation(app, ui);
+ });
+ if esc || stop_clicked {
cancel(app);
}
- ui.horizontal(|ui| {
- add_menu(app, ui);
- // A message waits for files still being read.
- let reading = !app.agent_chat.loading.is_empty();
- let send_b = ui
- .add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
- .on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
- crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
- // Files alone make a message.
- let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
- if reading {
- // Enter waits with the button.
- } else if (send_b.clicked() || enter || steer) && something {
- let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
- send(app, ui.ctx(), text);
- if steer {
- send_now(app, ui.ctx());
- }
- } else if steer {
- // Nothing typed: Ctrl+Enter sends what is already queued.
+ // A message waits for files still being read, and files alone make a message.
+ let reading = !app.agent_chat.loading.is_empty();
+ let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
+ if reading {
+ // Enter waits with the button.
+ } else if (send_clicked || enter || steer) && something {
+ let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
+ send(app, ui.ctx(), text);
+ if steer {
send_now(app, ui.ctx());
}
- if busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked() {
- cancel(app);
- }
- // A picture of the view goes with the message to agents that take pictures.
- let images = app.agent_chat.images || !state.running();
- let attach = ui
- .add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view")))
- .on_hover_text(tr("ui.agent_chat_attach_view_tip"))
- .on_disabled_hover_text(tr("ui.agent_chat_no_images"));
- crate::scenario::publish_handle("agent_chat:attach", attach.rect.center());
- });
+ } else if steer {
+ // Nothing typed: Ctrl+Enter sends what is already queued.
+ send_now(app, ui.ctx());
+ }
}
#[cfg(not(target_arch = "wasm32"))]
@@ -789,17 +897,23 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
// Allow (true), Deny (false), Always allow (None).
let mut approval: Option<(u64, Option<bool>)> = None;
let mut undo: Option<usize> = None;
+ let mut open_gallery: Option<Gallery> = None;
let current_doc = app.session.doc.clone();
for (n, item) in app.agent_chat.items.iter().enumerate() {
match item {
- Item::User { text, image, files } => {
+ Item::User { text, image, pictures, files } => {
ui.add_space(6.0);
egui::Frame::NONE.fill(t.accent_soft).corner_radius(6.0).inner_margin(egui::Margin::same(6)).show(ui, |ui| {
ui.set_width(ui.available_width());
if !text.is_empty() {
ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap());
}
- if *image {
+ if !pictures.is_empty() {
+ if let Some(at) = picture_strip(ui, n, pictures) {
+ open_gallery = Some(Gallery { pictures: pictures.clone(), at });
+ }
+ } else if *image {
+ // Taken when the agent starts on the message: nothing to show yet.
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
}
files_line(ui, files.iter().map(String::as_str));
@@ -1001,6 +1115,9 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
if send_queue_clicked {
send_now(app, ui.ctx());
}
+ if open_gallery.is_some() {
+ app.agent_chat.gallery = open_gallery;
+ }
if let Some((n, o)) = answer {
answer_permission(app, n, o.as_deref());
}
@@ -1027,13 +1144,6 @@ fn files_line<'a>(ui: &mut egui::Ui, names: impl Iterator<Item = &'a str>) {
/// The widest a file chip gets; a narrower chat makes them narrower.
const CHIP_W: f32 = 170.0;
-/// How many rows `n` chips take in a chat `avail` wide (`gap` between them).
-fn chip_rows(n: usize, avail: f32, gap: f32) -> usize {
- let avail = avail.max(1.0);
- let per_row = (((avail + gap) / (CHIP_W.min(avail) + gap)).floor() as usize).max(1);
- n.div_ceil(per_row)
-}
-
/// The files for the next message, as chips in rows that fit the chat: a picture's thumbnail (or
/// a file mark), the name cut to fit, and a × that takes the file off. Laid out by hand: egui
/// wraps plain widgets, not framed groups, and one row wider than the chat widens all below it.
diff --git a/crates/ui-egui/src/agent_chat/tests.rs b/crates/ui-egui/src/agent_chat/tests.rs
index 9da7d93729..6881b0da77 100644
--- a/crates/ui-egui/src/agent_chat/tests.rs
+++ b/crates/ui-egui/src/agent_chat/tests.rs
@@ -26,12 +26,12 @@ fn streamed_text_joins_and_tool_updates_change_their_call() {
#[test]
fn a_sign_in_request_keeps_the_question_for_after_the_login() {
let mut c = chat();
- c.last = Some(("make a cube".into(), false, vec![attach::Attachment::imported("motor.step".into())]));
+ c.last = Some(("make a cube".into(), false, None, vec![attach::Attachment::imported("motor.step".into())]));
c.state = Some(State::Busy);
assert!(c.on_event(Event::TurnEnded { reason: "auth".into() }).is_some());
c.on_event(Event::AuthRequired { methods: vec![] });
assert_eq!(c.state(), State::NeedsAuth);
- let (text, image, files) = c.queued.clone().expect("queued for after the login");
+ let (text, image, _, files) = c.queued.clone().expect("queued for after the login");
assert_eq!((text.as_str(), image), ("make a cube", false));
assert_eq!(files.iter().map(|f| f.name.as_str()).collect::<Vec<_>>(), ["motor.step"], "its files go again too");
}
@@ -39,7 +39,7 @@ fn a_sign_in_request_keeps_the_question_for_after_the_login() {
#[test]
fn a_plan_update_replaces_the_turns_plan() {
let mut c = chat();
- c.push(Item::User { text: "go".into(), image: false, files: Vec::new() });
+ c.push(Item::User { text: "go".into(), image: false, pictures: Vec::new(), files: Vec::new() });
c.on_event(Event::Plan(vec![("a".into(), "pending".into())]));
c.on_event(Event::Text { thought: false, text: "working".into() });
c.on_event(Event::Plan(vec![("a".into(), "completed".into())]));
@@ -68,7 +68,7 @@ fn app_in_a_turn() -> (SolveApp, Sender<ControlRequest>) {
let (tx, rx) = channel();
app.agent_chat.tools_rx = Some(rx);
app.agent_chat.state = Some(State::Busy);
- start_turn(&mut app, "two boxes".into(), false, Vec::new());
+ start_turn(&mut app, "two boxes".into(), false, None, Vec::new());
(app, tx)
}
@@ -1380,3 +1380,164 @@ fn an_agents_markdown_is_drawn_as_blocks() {
h.app.agent_chat.items.push(Item::Agent { thought: false, text: "*".repeat(50_000) + &"[".repeat(5_000) + &"`a".repeat(5_000) });
h.frames(3);
}
+
+/// A long message scrolls inside its box: the buttons under the box stay in the window (#2224).
+#[test]
+fn a_long_message_keeps_the_buttons_in_the_window() {
+ let mut h = crate::scenario::Harness::new();
+ h.call("ui.resize", json!({"width": 1600.0, "height": 1000.0}));
+ h.call("ui.agent_chat", json!({"open": true}));
+ h.frames(8);
+ let [_, rest] = h.widget("Send").expect("the Send button is drawn");
+ let box_before = h.app.agent_chat.panel_rect.map(|r| r.bottom());
+ h.app.agent_chat.input = "Please make the plate thicker and then add a fillet on every outside edge.\n".repeat(60);
+ h.frames(8);
+ let window = 1000.0;
+ for name in ["Send", "Attach view"] {
+ let [_, y] = h.widget(name).unwrap_or_else(|| panic!("{name} is drawn"));
+ assert!(y < window, "{name} is at {y}, below the window's bottom at {window}");
+ // The box is pinned to the bottom and grows upwards: the buttons do not move.
+ assert!((y - rest).abs() < 1.0, "{name} moved from {rest} to {y}");
+ }
+ assert_eq!(h.app.agent_chat.panel_rect.map(|r| r.bottom()), box_before, "the panel keeps its size");
+}
+
+fn test_png(colour: [u8; 4], width: u32) -> Vec<u8> {
+ let mut png = Vec::new();
+ let image = image::RgbaImage::from_pixel(width, 4, image::Rgba(colour));
+ image::DynamicImage::ImageRgba8(image).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).expect("a PNG");
+ png
+}
+
+/// A message's pictures show as thumbnails; a click opens the viewer on that one, where Next, the
+/// arrow keys and Esc work (#2224). A message whose picture is not taken yet keeps a plain caption.
+#[test]
+fn a_messages_pictures_open_in_a_viewer() {
+ let mut h = crate::scenario::Harness::new();
+ h.call("ui.agent_chat", json!({"open": true}));
+ h.app.agent_chat.items.push(Item::User { text: "waiting".into(), image: true, pictures: Vec::new(), files: Vec::new() });
+ h.frames(3);
+ let [x, y] = h.widget("View attached").expect("the plain caption is drawn");
+ h.call("ui.click", json!({"x": x, "y": y}));
+ h.frames(3);
+ assert!(h.app.agent_chat.gallery.is_none(), "no picture yet: nothing opens");
+
+ h.app.agent_chat.items.clear();
+ let pictures = vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view"), Picture::new(test_png([30, 30, 200, 255], 16), "")];
+ h.app.agent_chat.items.push(Item::User { text: "look".into(), image: true, pictures, files: Vec::new() });
+ h.frames(4);
+ assert!(h.widget("View attached").is_none(), "thumbnails replace the caption");
+ let point = |name: &str| {
+ let p = crate::scenario::handle_at(name).unwrap_or_else(|| panic!("{name} is drawn"));
+ json!({"x": p.x, "y": p.y})
+ };
+ // Hovering is not clicking.
+ let second = point("agent_chat:picture:0:1");
+ h.call("ui.move", second.clone());
+ h.frames(3);
+ assert!(h.app.agent_chat.gallery.is_none());
+ h.call("ui.click", second);
+ h.frames(4);
+ assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1), "the click opens the viewer on that picture");
+ assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "the picture is drawn big");
+ // Next wraps round; the arrow keys step too.
+ let next = point("agent_chat:gallery_next");
+ h.call("ui.click", next);
+ h.frames(3);
+ assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(0));
+ h.call("ui.key", json!({"key": "ArrowRight"}));
+ h.frames(3);
+ assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1));
+ let state = h.call("ui.agent_chat", json!({}));
+ assert_eq!(state["result"]["items"][0]["pictures"], 2);
+ h.call("ui.key", json!({"key": "Escape"}));
+ h.frames(3);
+ assert!(h.app.agent_chat.gallery.is_none(), "Esc closes the viewer");
+ // New chat closes it with the conversation.
+ h.app.agent_chat.gallery = Some(Gallery { pictures: vec![Picture::new(test_png([0, 0, 0, 255], 4), "")], at: 0 });
+ super::new_chat(&mut h.app);
+ assert!(h.app.agent_chat.gallery.is_none());
+}
+
+/// A message's picture is kept on exactly the message it was taken for: not an earlier one that
+/// also wanted a picture, not one that has it already, and nothing that isn't such a message.
+#[test]
+fn the_picture_goes_to_the_message_it_was_taken_for() {
+ let mut chat = AgentChat::default();
+ chat.push(Item::User { text: "no picture".into(), image: false, pictures: Vec::new(), files: Vec::new() });
+ chat.push(Item::User { text: "first".into(), image: true, pictures: vec![Picture::new(vec![1], "")], files: Vec::new() });
+ chat.push(Item::User { text: "second".into(), image: true, pictures: Vec::new(), files: Vec::new() });
+ chat.push(Item::User { text: "third".into(), image: true, pictures: Vec::new(), files: Vec::new() });
+ let sizes = |chat: &AgentChat| -> Vec<usize> {
+ chat.items
+ .iter()
+ .filter_map(|i| match i {
+ Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.png.len()).sum()),
+ _ => None,
+ })
+ .collect()
+ };
+ // The picture of the third message goes to the third, though the second still waits for one.
+ chat.keep_picture(Some(3), vec![2, 3]);
+ assert_eq!(sizes(&chat), [0, 1, 0, 2]);
+ // The same message again (it was sent again): nothing is added, nor moved to an older message.
+ chat.keep_picture(Some(3), vec![9; 5]);
+ assert_eq!(sizes(&chat), [0, 1, 0, 2]);
+ // A message that did not want one, no message, or a place that is gone.
+ chat.keep_picture(Some(0), vec![1]);
+ chat.keep_picture(None, vec![1]);
+ chat.keep_picture(Some(99), vec![1]);
+ assert_eq!(sizes(&chat), [0, 1, 0, 2]);
+ chat.keep_picture(Some(2), vec![0; super::MAX_PICTURE_BYTES + 1]);
+ assert_eq!(sizes(&chat), [0, 1, 0, 2], "an oversized picture is not kept");
+}
+
+/// A message sent again after a sign-in is still the message it was: its turn points at its own
+/// place in the conversation, so its picture is kept there, once, and never on an older message that
+/// wanted one (#2224 review). The picture itself can't be drawn without a renderer in a test, so the
+/// place a turn is started for is what is checked.
+#[test]
+fn a_message_sent_again_after_sign_in_stays_the_message_it_was() {
+ let (mut app, script, ctx) = app_with_script();
+ app.agent_chat.images = true;
+ // An older message that wanted a picture and has none.
+ app.agent_chat.push(Item::User { text: "older".into(), image: true, pictures: Vec::new(), files: Vec::new() });
+ app.agent_chat.attach_view = true;
+ send(&mut app, &ctx, "the new one".into());
+ let place = app.agent_chat.items.len() - 1;
+ assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "the turn knows its message");
+ // The agent asked for a sign-in; after it the message goes again, as it is done on a sign-in.
+ ended(&script, "");
+ poll(&mut app, &ctx);
+ app.agent_chat.queued = app.agent_chat.last.take();
+ script.report(Event::Ready { agent: "fake".into(), images: true });
+ poll(&mut app, &ctx);
+ assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "sent again, still for the same message");
+ assert_eq!(app.agent_chat.items.iter().filter(|i| matches!(i, Item::User { .. })).count(), 2, "not a new message");
+ // Its picture goes there, once; the older message gets none.
+ app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4));
+ app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4));
+ let counts: Vec<usize> =
+ app.agent_chat.items.iter().filter_map(|i| if let Item::User { pictures, .. } = i { Some(pictures.len()) } else { None }).collect();
+ assert_eq!(counts, [0, 1]);
+}
+
+/// The picture viewer opens over the chat's own window too, not only over the docked panel (#1745).
+#[test]
+fn the_viewer_opens_in_the_popped_out_window() {
+ let mut h = crate::scenario::Harness::new();
+ h.call("ui.agent_chat", json!({"open": true}));
+ h.app.preferences.agent_chat_popped = true;
+ h.app.agent_chat.items.push(Item::User {
+ text: "look".into(),
+ image: true,
+ pictures: vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view")],
+ files: Vec::new(),
+ });
+ h.frames(4);
+ let at = crate::scenario::handle_at("agent_chat:picture:0:0").expect("the thumbnail is drawn in the window");
+ h.call("ui.click", json!({"x": at.x, "y": at.y}));
+ h.frames(4);
+ assert!(h.app.agent_chat.gallery.is_some(), "the click opened the viewer");
+ assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "and it is drawn");
+}
diff --git a/docs/agent-chat.md b/docs/agent-chat.md
index c9e39630ff..9a03b6908b 100644
--- a/docs/agent-chat.md
+++ b/docs/agent-chat.md
@@ -195,7 +195,11 @@ The agent reaches the open design only through SolidCraft's own tools (the
(with **Copy**), bold, italic, `code` and links (web links only; each link is a button that opens
it in your browser). What you type, and the agent's thoughts, stay plain text.
- **Attach view:** tick it to send a picture of the current view with your message, for agents
- that take pictures.
+ that take pictures. The picture is taken when the agent starts on the message and then shows as
+ a small thumbnail under it. Hold the pointer over the thumbnail for a bigger preview that stays
+ while you do; click it to open the viewer (the message's pictures one at a time, with **‹** and
+ **›** or the arrow keys, closed with Esc or a click outside). A long message scrolls inside its
+ box, so **Send** and the buttons beside it stay in view.
## Permissions