Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

Agent chat: attached pictures use the shared viewer, in any format, decoded in the background (#2554)

Rec0iLdeb28dd72119
diff --git a/crates/ui-egui/src/agent_chat/mod.rs b/crates/ui-egui/src/agent_chat/mod.rs
index 88d62f820b..3347c9ff88 100644
--- a/crates/ui-egui/src/agent_chat/mod.rs
+++ b/crates/ui-egui/src/agent_chat/mod.rs
@@ -95,41 +95,86 @@ impl State {
}
}
-/// A picture shown with a message: what the agent got (a picture of the view, or any other
-/// attachment) as PNG, and a note shown with it in the viewer. The thumbnail and the viewer share
-/// one texture, made when the picture is first drawn.
+/// A picture shown with a message: what the agent got (a picture of the view, or a file the user
+/// attached) as the encoded bytes it was sent in (PNG, JPEG, GIF, WebP…), and a note shown with it
+/// (the file's name). The thumbnail and the viewer share one texture, decoded off the window
+/// thread the first time the picture is drawn.
#[derive(Clone)]
pub struct Picture {
- png: Arc<Vec<u8>>,
+ bytes: Arc<Vec<u8>>,
note: String,
- texture: Arc<std::sync::OnceLock<Option<egui::TextureHandle>>>,
+ /// The picture of the view that Attach view sends, not a file.
+ view: bool,
+ decode: Arc<Decode>,
+}
+
+/// A picture's decoding: started once, and the texture (`None`: not a picture) once it is in.
+#[derive(Default)]
+struct Decode {
+ started: std::sync::atomic::AtomicBool,
+ texture: std::sync::OnceLock<Option<egui::TextureHandle>>,
+}
+
+/// How a picture can be drawn now.
+#[cfg(not(target_arch = "wasm32"))]
+pub enum Shown {
+ /// Still being decoded: a repaint comes when it is in.
+ Loading,
+ Ready(egui::TextureHandle),
+ /// The bytes are not a picture the app can read.
+ Failed,
}
impl Picture {
- pub fn new(png: Vec<u8>, note: impl Into<String>) -> Self {
- Self { png: Arc::new(png), note: note.into(), texture: Arc::default() }
+ pub fn new(bytes: Vec<u8>, note: impl Into<String>) -> Self {
+ Self::shared(Arc::new(bytes), note, false)
+ }
+
+ fn shared(bytes: Arc<Vec<u8>>, note: impl Into<String>, view: bool) -> Self {
+ Self { bytes, note: note.into(), view, decode: Arc::default() }
}
- /// The picture as a texture, `None` when it can't be decoded.
+ /// The picture as a texture, decoded in the background on first use.
#[cfg(not(target_arch = "wasm32"))]
- fn texture(&self, ctx: &egui::Context) -> Option<egui::TextureHandle> {
- self.texture
- .get_or_init(|| {
- let rgba = image::load_from_memory_with_format(&self.png, image::ImageFormat::Png).ok()?.to_rgba8();
- let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?];
- Some(ctx.load_texture(
- "sc_agent_chat_picture",
- egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()),
- egui::TextureOptions::LINEAR,
- ))
- })
- .clone()
+ fn texture(&self, ctx: &egui::Context) -> Shown {
+ use std::sync::atomic::Ordering;
+ if let Some(done) = self.decode.texture.get() {
+ return done.clone().map_or(Shown::Failed, Shown::Ready);
+ }
+ if !self.decode.started.swap(true, Ordering::AcqRel) {
+ let (bytes, decode, ctx) = (Arc::clone(&self.bytes), Arc::clone(&self.decode), ctx.clone());
+ let spawned = std::thread::Builder::new().name("agent-chat-picture".into()).spawn(move || {
+ let texture = decode_picture(&bytes).map(|image| ctx.load_texture("sc_agent_chat_picture", image, egui::TextureOptions::LINEAR));
+ let _ = decode.texture.set(texture);
+ ctx.request_repaint();
+ });
+ if spawned.is_err() {
+ let _ = self.decode.texture.set(None);
+ return Shown::Failed;
+ }
+ }
+ Shown::Loading
}
}
+/// Decode a picture in any format the app reads, within limits (it came through the attachment
+/// checks already; a hostile header still can't ask for gigabytes here).
+#[cfg(not(target_arch = "wasm32"))]
+fn decode_picture(bytes: &[u8]) -> Option<egui::ColorImage> {
+ let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().ok()?;
+ let mut limits = image::Limits::default();
+ limits.max_image_width = Some(20_000);
+ limits.max_image_height = Some(20_000);
+ limits.max_alloc = Some(1 << 30);
+ reader.limits(limits);
+ let rgba = reader.decode().ok()?.to_rgba8();
+ let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?];
+ Some(egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()))
+}
+
impl std::fmt::Debug for Picture {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "Picture({} bytes, {:?})", self.png.len(), self.note)
+ write!(f, "Picture({} bytes, {:?})", self.bytes.len(), self.note)
}
}
@@ -404,10 +449,10 @@ impl AgentChat {
/// never to another message.
fn keep_picture(&mut self, at: Option<usize>, png: Vec<u8>) {
if let Some(Item::User { image: true, pictures, .. }) = at.and_then(|i| self.items.get_mut(i))
- && pictures.is_empty()
+ && !pictures.iter().any(|p| p.view)
&& png.len() <= MAX_PICTURE_BYTES
{
- pictures.push(Picture::new(png, ""));
+ pictures.push(Picture::shared(Arc::new(png), "", true));
}
}
fn notice(&mut self, text: impl Into<String>, error: bool) {
@@ -767,7 +812,21 @@ pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) {
/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it
/// cut in on its previous step).
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec<attach::Attachment>, interrupted: bool) {
- app.agent_chat.push(Item::User { text: text.clone(), image, pictures: Vec::new(), files: files.iter().map(|f| f.name.clone()).collect() });
+ // Attached pictures show as thumbnails with the viewer, as the view's picture does; the other
+ // files are listed by name.
+ let pictures = files
+ .iter()
+ .filter_map(|f| match &f.content {
+ attach::Content::Picture { bytes, .. } => Some(Picture::shared(Arc::clone(bytes), f.name.clone(), false)),
+ _ => None,
+ })
+ .collect();
+ app.agent_chat.push(Item::User {
+ text: text.clone(),
+ image,
+ pictures,
+ files: files.iter().filter(|f| !f.is_picture()).map(|f| f.name.clone()).collect(),
+ });
// The message's place in the conversation: its picture is kept there when it is taken.
let at = app.agent_chat.items.len().checked_sub(1);
match app.agent_chat.state() {
diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rs
index 66358816c6..93344ba31c 100644
--- a/crates/ui-egui/src/agent_chat/panel.rs
+++ b/crates/ui-egui/src/agent_chat/panel.rs
@@ -129,9 +129,19 @@ fn picture_strip(ui: &mut egui::Ui, n: usize, pictures: &[Picture]) -> Option<us
let mut clicked = None;
ui.horizontal_wrapped(|ui| {
for (i, picture) in pictures.iter().enumerate() {
- let Some(texture) = picture.texture(ui.ctx()) else {
- ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
- continue;
+ let texture = match picture.texture(ui.ctx()) {
+ Shown::Ready(texture) => texture,
+ // Being decoded: a spinner the size of a thumbnail's corner, until it is in.
+ Shown::Loading => {
+ ui.add(egui::Spinner::new().size(THUMB_MAX.1 / 2.0));
+ continue;
+ }
+ // Not a readable picture: its name, or the plain caption.
+ Shown::Failed => {
+ let caption = if picture.note.is_empty() { tr("ui.agent_chat_view_attached").to_string() } else { picture.note.clone() };
+ ui.label(RichText::new(caption).color(t.text_dim).size(11.0));
+ continue;
+ }
};
let size = fit(texture.size_vec2(), vec2(THUMB_MAX.0, THUMB_MAX.1));
let thumb = ui.add(egui::Image::new((texture.id(), size)).corner_radius(4.0).sense(Sense::click()));
@@ -186,9 +196,17 @@ fn gallery_window(app: &mut SolveApp, ctx: &egui::Context) {
ui.label(RichText::new(format!("{} / {}", gallery.at + 1, count)).color(t.text_dim));
});
if let Some(picture) = gallery.pictures.get(gallery.at) {
- if let Some(texture) = picture.texture(ui.ctx()) {
- let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0));
- crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect);
+ match picture.texture(ui.ctx()) {
+ Shown::Ready(texture) => {
+ let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0));
+ crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect);
+ }
+ Shown::Loading => {
+ ui.add(egui::Spinner::new().size(32.0));
+ }
+ Shown::Failed => {
+ ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim));
+ }
}
if !picture.note.is_empty() {
ui.add(egui::Label::new(&picture.note).wrap());
diff --git a/crates/ui-egui/src/agent_chat/tests.rs b/crates/ui-egui/src/agent_chat/tests.rs
index 6881b0da77..95756ff92c 100644
--- a/crates/ui-egui/src/agent_chat/tests.rs
+++ b/crates/ui-egui/src/agent_chat/tests.rs
@@ -932,7 +932,10 @@ fn a_message_carries_its_files() {
assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm, Pi 5 inside"), "{text}");
assert_eq!(sent[0].iter().filter(|p| matches!(p, PromptPart::Image { mime, .. } if mime == "image/png")).count(), 1);
assert!(app.agent_chat.attachments.is_empty(), "the files went with the message");
- assert_eq!(status(&app)["items"][0]["files"], json!(["target.png", "brief.md"]));
+ // The picture shows as a thumbnail with the viewer (its name is the note); the text file by name.
+ assert_eq!(status(&app)["items"][0]["files"], json!(["brief.md"]));
+ assert_eq!(status(&app)["items"][0]["pictures"], 1);
+ assert!(matches!(&app.agent_chat.items[0], Item::User { pictures, .. } if pictures.first().is_some_and(|p| p.note == "target.png" && !p.view)));
let _ = std::fs::remove_dir_all(&d);
}
@@ -1054,7 +1057,8 @@ fn a_queued_message_keeps_its_files_through_stop_and_sends_them() {
assert!(text.starts_with("like this") && !text.contains("\n\n\n\n"), "a message of files alone adds no empty paragraph: {text:?}");
assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm"), "{text}");
assert_eq!(pictures_in(last), 1);
- assert_eq!(status(&app)["items"].as_array().and_then(|i| i.last()).map(|i| i["files"].clone()), Some(json!(["target.png", "brief.md"])));
+ let sent = status(&app)["items"].as_array().and_then(|i| i.last()).cloned().unwrap_or_default();
+ assert_eq!((sent["files"].clone(), sent["pictures"].clone()), (json!(["brief.md"]), json!(1)));
let _ = std::fs::remove_dir_all(&d);
}
@@ -1424,8 +1428,13 @@ fn a_messages_pictures_open_in_a_viewer() {
h.app.agent_chat.items.clear();
let pictures = vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view"), Picture::new(test_png([30, 30, 200, 255], 16), "")];
+ let ours = pictures.clone();
h.app.agent_chat.items.push(Item::User { text: "look".into(), image: true, pictures, files: Vec::new() });
- h.frames(4);
+ h.frames(2);
+ // Decoded in the background: let both finish before looking for thumbnails.
+ for picture in &ours {
+ assert!(until_decoded(&mut h, picture), "the picture was decoded");
+ }
assert!(h.widget("View attached").is_none(), "thumbnails replace the caption");
let point = |name: &str| {
let p = crate::scenario::handle_at(name).unwrap_or_else(|| panic!("{name} is drawn"));
@@ -1472,7 +1481,7 @@ fn the_picture_goes_to_the_message_it_was_taken_for() {
chat.items
.iter()
.filter_map(|i| match i {
- Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.png.len()).sum()),
+ Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.bytes.len()).sum()),
_ => None,
})
.collect()
@@ -1534,10 +1543,82 @@ fn the_viewer_opens_in_the_popped_out_window() {
pictures: vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view")],
files: Vec::new(),
});
- h.frames(4);
+ h.frames(2);
+ let Item::User { pictures, .. } = &h.app.agent_chat.items[0] else { panic!("the message") };
+ let first = pictures[0].clone();
+ assert!(until_decoded(&mut h, &first), "the picture was decoded");
let at = crate::scenario::handle_at("agent_chat:picture:0:0").expect("the thumbnail is drawn in the window");
h.call("ui.click", json!({"x": at.x, "y": at.y}));
h.frames(4);
assert!(h.app.agent_chat.gallery.is_some(), "the click opened the viewer");
assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "and it is drawn");
}
+
+// ---- Attached pictures in the shared viewer ----
+
+/// A small picture encoded as `format`.
+fn encoded(format: image::ImageFormat, width: u32) -> Vec<u8> {
+ let mut out = Vec::new();
+ let image = image::RgbaImage::from_pixel(width, 4, image::Rgba([40, 160, 90, 255]));
+ let image = if format == image::ImageFormat::Jpeg {
+ image::DynamicImage::ImageRgb8(image::DynamicImage::ImageRgba8(image).to_rgb8())
+ } else {
+ image::DynamicImage::ImageRgba8(image)
+ };
+ image.write_to(&mut std::io::Cursor::new(&mut out), format).expect("encoded");
+ out
+}
+
+/// Frames until a picture is decoded (it is, in the background, on first draw), at most two seconds.
+fn until_decoded(h: &mut crate::scenario::Harness, picture: &Picture) -> bool {
+ for _ in 0..200 {
+ h.frames(1);
+ if picture.decode.texture.get().is_some() {
+ h.frames(2);
+ return true;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ }
+ false
+}
+
+/// Pictures in any format the attachments take are drawn: JPEG and GIF as well as PNG, decoded off
+/// the window thread (the first frames show a spinner), one that isn't a picture says so.
+#[test]
+fn attached_pictures_of_any_format_are_decoded_in_the_background() {
+ let mut h = crate::scenario::Harness::new();
+ h.call("ui.agent_chat", json!({"open": true}));
+ for (format, width) in [(image::ImageFormat::Png, 8), (image::ImageFormat::Jpeg, 16), (image::ImageFormat::Gif, 12)] {
+ let picture = Picture::new(encoded(format, width), format!("photo.{format:?}"));
+ h.app.agent_chat.items.clear();
+ h.app.agent_chat.items.push(Item::User { text: "look".into(), image: false, pictures: vec![picture.clone()], files: Vec::new() });
+ h.frames(2);
+ assert!(until_decoded(&mut h, &picture), "{format:?} was decoded");
+ let texture = picture.decode.texture.get().and_then(|t| t.as_ref()).unwrap_or_else(|| panic!("{format:?} is a texture"));
+ assert_eq!(texture.size()[0], width as usize, "{format:?} keeps its width");
+ assert!(crate::scenario::handle_at("agent_chat:picture:0:0").is_some(), "{format:?} has its thumbnail");
+ }
+ // Bytes that are no picture: the name where the thumbnail would be, and nothing breaks.
+ let broken = Picture::new(b"not a picture at all".to_vec(), "fake.png");
+ h.app.agent_chat.items.clear();
+ h.app.agent_chat.items.push(Item::User { text: "look".into(), image: false, pictures: vec![broken.clone()], files: Vec::new() });
+ assert!(until_decoded(&mut h, &broken));
+ assert!(matches!(broken.decode.texture.get(), Some(None)), "it could not be read");
+ assert!(h.widget("fake.png").is_some(), "its name stands in for the thumbnail");
+}
+
+/// A picture attached as a file goes to the viewer like the view's own picture, named in it, and
+/// the view's picture is added beside it rather than replacing it, once.
+#[test]
+fn an_attached_picture_and_the_views_picture_share_the_viewer() {
+ let mut chat = AgentChat::default();
+ let attached = Picture::shared(Arc::new(encoded(image::ImageFormat::Jpeg, 16)), "photo.jpg", false);
+ chat.push(Item::User { text: "see".into(), image: true, pictures: vec![attached], files: Vec::new() });
+ chat.keep_picture(Some(0), encoded(image::ImageFormat::Png, 8));
+ let Some(Item::User { pictures, .. }) = chat.items.last() else { panic!("a sent message") };
+ assert_eq!(pictures.iter().map(|p| (p.note.as_str(), p.view)).collect::<Vec<_>>(), [("photo.jpg", false), ("", true)]);
+ // A second view picture is not added to the same message.
+ chat.keep_picture(Some(0), encoded(image::ImageFormat::Png, 8));
+ let Some(Item::User { pictures, .. }) = chat.items.last() else { panic!("a sent message") };
+ assert_eq!(pictures.len(), 2);
+}
diff --git a/docs/agent-chat.md b/docs/agent-chat.md
index 9a03b6908b..c5f14def8f 100644
--- a/docs/agent-chat.md
+++ b/docs/agent-chat.md
@@ -81,8 +81,12 @@ sketch, a brief, a part to fit around.
shows a spinner meanwhile, and **Send** waits), and the message tells the agent it was added; a
failed import says why and the agent is told nothing of it. Taking a part's chip off the message
leaves the part in the design (Undo removes it).
-- Every file is named in the message, and listed under it in the conversation. A file that
- can't be attached says why.
+- Every file is named in the message. In the conversation the attached pictures show under it as
+ thumbnails (hold the pointer over one for a bigger preview, click it to open the viewer, where
+ the file's name is shown and ‹ › step through the message's pictures, together with the picture
+ of the view); the other files are listed by name. Pictures in any of the formats above are
+ decoded in the background, with a spinner until they are in. A file that can't be attached says
+ why.
Files stay with their message: one queued while the agent works keeps its files, and one sent
again after a sign-in sends them again. When several queued messages go as one prompt, their