SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
Agent chat: attached pictures use the shared viewer, in any format, decoded in the background (#2554)
Rec0iLdeb28dd72119
diff --git a/crates/ui-egui/src/agent_chat/mod.rs b/crates/ui-egui/src/agent_chat/mod.rsindex 88d62f820b..3347c9ff88 100644--- a/crates/ui-egui/src/agent_chat/mod.rs+++ b/crates/ui-egui/src/agent_chat/mod.rs@@ -95,41 +95,86 @@ impl State {}}-/// A picture shown with a message: what the agent got (a picture of the view, or any other-/// attachment) as PNG, and a note shown with it in the viewer. The thumbnail and the viewer share-/// one texture, made when the picture is first drawn.+/// A picture shown with a message: what the agent got (a picture of the view, or a file the user+/// attached) as the encoded bytes it was sent in (PNG, JPEG, GIF, WebP…), and a note shown with it+/// (the file's name). The thumbnail and the viewer share one texture, decoded off the window+/// thread the first time the picture is drawn.#[derive(Clone)]pub struct Picture {- png: Arc<Vec<u8>>,+ bytes: Arc<Vec<u8>>,note: String,- texture: Arc<std::sync::OnceLock<Option<egui::TextureHandle>>>,+ /// The picture of the view that Attach view sends, not a file.+ view: bool,+ decode: Arc<Decode>,+}++/// A picture's decoding: started once, and the texture (`None`: not a picture) once it is in.+#[derive(Default)]+struct Decode {+ started: std::sync::atomic::AtomicBool,+ texture: std::sync::OnceLock<Option<egui::TextureHandle>>,+}++/// How a picture can be drawn now.+#[cfg(not(target_arch = "wasm32"))]+pub enum Shown {+ /// Still being decoded: a repaint comes when it is in.+ Loading,+ Ready(egui::TextureHandle),+ /// The bytes are not a picture the app can read.+ Failed,}impl Picture {- pub fn new(png: Vec<u8>, note: impl Into<String>) -> Self {- Self { png: Arc::new(png), note: note.into(), texture: Arc::default() }+ pub fn new(bytes: Vec<u8>, note: impl Into<String>) -> Self {+ Self::shared(Arc::new(bytes), note, false)+ }++ fn shared(bytes: Arc<Vec<u8>>, note: impl Into<String>, view: bool) -> Self {+ Self { bytes, note: note.into(), view, decode: Arc::default() }}- /// The picture as a texture, `None` when it can't be decoded.+ /// The picture as a texture, decoded in the background on first use.#[cfg(not(target_arch = "wasm32"))]- fn texture(&self, ctx: &egui::Context) -> Option<egui::TextureHandle> {- self.texture- .get_or_init(|| {- let rgba = image::load_from_memory_with_format(&self.png, image::ImageFormat::Png).ok()?.to_rgba8();- let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?];- Some(ctx.load_texture(- "sc_agent_chat_picture",- egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()),- egui::TextureOptions::LINEAR,- ))- })- .clone()+ fn texture(&self, ctx: &egui::Context) -> Shown {+ use std::sync::atomic::Ordering;+ if let Some(done) = self.decode.texture.get() {+ return done.clone().map_or(Shown::Failed, Shown::Ready);+ }+ if !self.decode.started.swap(true, Ordering::AcqRel) {+ let (bytes, decode, ctx) = (Arc::clone(&self.bytes), Arc::clone(&self.decode), ctx.clone());+ let spawned = std::thread::Builder::new().name("agent-chat-picture".into()).spawn(move || {+ let texture = decode_picture(&bytes).map(|image| ctx.load_texture("sc_agent_chat_picture", image, egui::TextureOptions::LINEAR));+ let _ = decode.texture.set(texture);+ ctx.request_repaint();+ });+ if spawned.is_err() {+ let _ = self.decode.texture.set(None);+ return Shown::Failed;+ }+ }+ Shown::Loading}}+/// Decode a picture in any format the app reads, within limits (it came through the attachment+/// checks already; a hostile header still can't ask for gigabytes here).+#[cfg(not(target_arch = "wasm32"))]+fn decode_picture(bytes: &[u8]) -> Option<egui::ColorImage> {+ let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().ok()?;+ let mut limits = image::Limits::default();+ limits.max_image_width = Some(20_000);+ limits.max_image_height = Some(20_000);+ limits.max_alloc = Some(1 << 30);+ reader.limits(limits);+ let rgba = reader.decode().ok()?.to_rgba8();+ let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?];+ Some(egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()))+}+impl std::fmt::Debug for Picture {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(f, "Picture({} bytes, {:?})", self.png.len(), self.note)+ write!(f, "Picture({} bytes, {:?})", self.bytes.len(), self.note)}}@@ -404,10 +449,10 @@ impl AgentChat {/// never to another message.fn keep_picture(&mut self, at: Option<usize>, png: Vec<u8>) {if let Some(Item::User { image: true, pictures, .. }) = at.and_then(|i| self.items.get_mut(i))- && pictures.is_empty()+ && !pictures.iter().any(|p| p.view)&& png.len() <= MAX_PICTURE_BYTES{- pictures.push(Picture::new(png, ""));+ pictures.push(Picture::shared(Arc::new(png), "", true));}}fn notice(&mut self, text: impl Into<String>, error: bool) {@@ -767,7 +812,21 @@ pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) {/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it/// cut in on its previous step).fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec<attach::Attachment>, interrupted: bool) {- app.agent_chat.push(Item::User { text: text.clone(), image, pictures: Vec::new(), files: files.iter().map(|f| f.name.clone()).collect() });+ // Attached pictures show as thumbnails with the viewer, as the view's picture does; the other+ // files are listed by name.+ let pictures = files+ .iter()+ .filter_map(|f| match &f.content {+ attach::Content::Picture { bytes, .. } => Some(Picture::shared(Arc::clone(bytes), f.name.clone(), false)),+ _ => None,+ })+ .collect();+ app.agent_chat.push(Item::User {+ text: text.clone(),+ image,+ pictures,+ files: files.iter().filter(|f| !f.is_picture()).map(|f| f.name.clone()).collect(),+ });// The message's place in the conversation: its picture is kept there when it is taken.let at = app.agent_chat.items.len().checked_sub(1);match app.agent_chat.state() {diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rsindex 66358816c6..93344ba31c 100644--- a/crates/ui-egui/src/agent_chat/panel.rs+++ b/crates/ui-egui/src/agent_chat/panel.rs@@ -129,9 +129,19 @@ fn picture_strip(ui: &mut egui::Ui, n: usize, pictures: &[Picture]) -> Option<uslet mut clicked = None;ui.horizontal_wrapped(|ui| {for (i, picture) in pictures.iter().enumerate() {- let Some(texture) = picture.texture(ui.ctx()) else {- ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));- continue;+ let texture = match picture.texture(ui.ctx()) {+ Shown::Ready(texture) => texture,+ // Being decoded: a spinner the size of a thumbnail's corner, until it is in.+ Shown::Loading => {+ ui.add(egui::Spinner::new().size(THUMB_MAX.1 / 2.0));+ continue;+ }+ // Not a readable picture: its name, or the plain caption.+ Shown::Failed => {+ let caption = if picture.note.is_empty() { tr("ui.agent_chat_view_attached").to_string() } else { picture.note.clone() };+ ui.label(RichText::new(caption).color(t.text_dim).size(11.0));+ continue;+ }};let size = fit(texture.size_vec2(), vec2(THUMB_MAX.0, THUMB_MAX.1));let thumb = ui.add(egui::Image::new((texture.id(), size)).corner_radius(4.0).sense(Sense::click()));@@ -186,9 +196,17 @@ fn gallery_window(app: &mut SolveApp, ctx: &egui::Context) {ui.label(RichText::new(format!("{} / {}", gallery.at + 1, count)).color(t.text_dim));});if let Some(picture) = gallery.pictures.get(gallery.at) {- if let Some(texture) = picture.texture(ui.ctx()) {- let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0));- crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect);+ match picture.texture(ui.ctx()) {+ Shown::Ready(texture) => {+ let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0));+ crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect);+ }+ Shown::Loading => {+ ui.add(egui::Spinner::new().size(32.0));+ }+ Shown::Failed => {+ ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim));+ }}if !picture.note.is_empty() {ui.add(egui::Label::new(&picture.note).wrap());diff --git a/crates/ui-egui/src/agent_chat/tests.rs b/crates/ui-egui/src/agent_chat/tests.rsindex 6881b0da77..95756ff92c 100644--- a/crates/ui-egui/src/agent_chat/tests.rs+++ b/crates/ui-egui/src/agent_chat/tests.rs@@ -932,7 +932,10 @@ fn a_message_carries_its_files() {assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm, Pi 5 inside"), "{text}");assert_eq!(sent[0].iter().filter(|p| matches!(p, PromptPart::Image { mime, .. } if mime == "image/png")).count(), 1);assert!(app.agent_chat.attachments.is_empty(), "the files went with the message");- assert_eq!(status(&app)["items"][0]["files"], json!(["target.png", "brief.md"]));+ // The picture shows as a thumbnail with the viewer (its name is the note); the text file by name.+ assert_eq!(status(&app)["items"][0]["files"], json!(["brief.md"]));+ assert_eq!(status(&app)["items"][0]["pictures"], 1);+ assert!(matches!(&app.agent_chat.items[0], Item::User { pictures, .. } if pictures.first().is_some_and(|p| p.note == "target.png" && !p.view)));let _ = std::fs::remove_dir_all(&d);}@@ -1054,7 +1057,8 @@ fn a_queued_message_keeps_its_files_through_stop_and_sends_them() {assert!(text.starts_with("like this") && !text.contains("\n\n\n\n"), "a message of files alone adds no empty paragraph: {text:?}");assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm"), "{text}");assert_eq!(pictures_in(last), 1);- assert_eq!(status(&app)["items"].as_array().and_then(|i| i.last()).map(|i| i["files"].clone()), Some(json!(["target.png", "brief.md"])));+ let sent = status(&app)["items"].as_array().and_then(|i| i.last()).cloned().unwrap_or_default();+ assert_eq!((sent["files"].clone(), sent["pictures"].clone()), (json!(["brief.md"]), json!(1)));let _ = std::fs::remove_dir_all(&d);}@@ -1424,8 +1428,13 @@ fn a_messages_pictures_open_in_a_viewer() {h.app.agent_chat.items.clear();let pictures = vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view"), Picture::new(test_png([30, 30, 200, 255], 16), "")];+ let ours = pictures.clone();h.app.agent_chat.items.push(Item::User { text: "look".into(), image: true, pictures, files: Vec::new() });- h.frames(4);+ h.frames(2);+ // Decoded in the background: let both finish before looking for thumbnails.+ for picture in &ours {+ assert!(until_decoded(&mut h, picture), "the picture was decoded");+ }assert!(h.widget("View attached").is_none(), "thumbnails replace the caption");let point = |name: &str| {let p = crate::scenario::handle_at(name).unwrap_or_else(|| panic!("{name} is drawn"));@@ -1472,7 +1481,7 @@ fn the_picture_goes_to_the_message_it_was_taken_for() {chat.items.iter().filter_map(|i| match i {- Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.png.len()).sum()),+ Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.bytes.len()).sum()),_ => None,}).collect()@@ -1534,10 +1543,82 @@ fn the_viewer_opens_in_the_popped_out_window() {pictures: vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view")],files: Vec::new(),});- h.frames(4);+ h.frames(2);+ let Item::User { pictures, .. } = &h.app.agent_chat.items[0] else { panic!("the message") };+ let first = pictures[0].clone();+ assert!(until_decoded(&mut h, &first), "the picture was decoded");let at = crate::scenario::handle_at("agent_chat:picture:0:0").expect("the thumbnail is drawn in the window");h.call("ui.click", json!({"x": at.x, "y": at.y}));h.frames(4);assert!(h.app.agent_chat.gallery.is_some(), "the click opened the viewer");assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "and it is drawn");}++// ---- Attached pictures in the shared viewer ----++/// A small picture encoded as `format`.+fn encoded(format: image::ImageFormat, width: u32) -> Vec<u8> {+ let mut out = Vec::new();+ let image = image::RgbaImage::from_pixel(width, 4, image::Rgba([40, 160, 90, 255]));+ let image = if format == image::ImageFormat::Jpeg {+ image::DynamicImage::ImageRgb8(image::DynamicImage::ImageRgba8(image).to_rgb8())+ } else {+ image::DynamicImage::ImageRgba8(image)+ };+ image.write_to(&mut std::io::Cursor::new(&mut out), format).expect("encoded");+ out+}++/// Frames until a picture is decoded (it is, in the background, on first draw), at most two seconds.+fn until_decoded(h: &mut crate::scenario::Harness, picture: &Picture) -> bool {+ for _ in 0..200 {+ h.frames(1);+ if picture.decode.texture.get().is_some() {+ h.frames(2);+ return true;+ }+ std::thread::sleep(std::time::Duration::from_millis(10));+ }+ false+}++/// Pictures in any format the attachments take are drawn: JPEG and GIF as well as PNG, decoded off+/// the window thread (the first frames show a spinner), one that isn't a picture says so.+#[test]+fn attached_pictures_of_any_format_are_decoded_in_the_background() {+ let mut h = crate::scenario::Harness::new();+ h.call("ui.agent_chat", json!({"open": true}));+ for (format, width) in [(image::ImageFormat::Png, 8), (image::ImageFormat::Jpeg, 16), (image::ImageFormat::Gif, 12)] {+ let picture = Picture::new(encoded(format, width), format!("photo.{format:?}"));+ h.app.agent_chat.items.clear();+ h.app.agent_chat.items.push(Item::User { text: "look".into(), image: false, pictures: vec![picture.clone()], files: Vec::new() });+ h.frames(2);+ assert!(until_decoded(&mut h, &picture), "{format:?} was decoded");+ let texture = picture.decode.texture.get().and_then(|t| t.as_ref()).unwrap_or_else(|| panic!("{format:?} is a texture"));+ assert_eq!(texture.size()[0], width as usize, "{format:?} keeps its width");+ assert!(crate::scenario::handle_at("agent_chat:picture:0:0").is_some(), "{format:?} has its thumbnail");+ }+ // Bytes that are no picture: the name where the thumbnail would be, and nothing breaks.+ let broken = Picture::new(b"not a picture at all".to_vec(), "fake.png");+ h.app.agent_chat.items.clear();+ h.app.agent_chat.items.push(Item::User { text: "look".into(), image: false, pictures: vec![broken.clone()], files: Vec::new() });+ assert!(until_decoded(&mut h, &broken));+ assert!(matches!(broken.decode.texture.get(), Some(None)), "it could not be read");+ assert!(h.widget("fake.png").is_some(), "its name stands in for the thumbnail");+}++/// A picture attached as a file goes to the viewer like the view's own picture, named in it, and+/// the view's picture is added beside it rather than replacing it, once.+#[test]+fn an_attached_picture_and_the_views_picture_share_the_viewer() {+ let mut chat = AgentChat::default();+ let attached = Picture::shared(Arc::new(encoded(image::ImageFormat::Jpeg, 16)), "photo.jpg", false);+ chat.push(Item::User { text: "see".into(), image: true, pictures: vec![attached], files: Vec::new() });+ chat.keep_picture(Some(0), encoded(image::ImageFormat::Png, 8));+ let Some(Item::User { pictures, .. }) = chat.items.last() else { panic!("a sent message") };+ assert_eq!(pictures.iter().map(|p| (p.note.as_str(), p.view)).collect::<Vec<_>>(), [("photo.jpg", false), ("", true)]);+ // A second view picture is not added to the same message.+ chat.keep_picture(Some(0), encoded(image::ImageFormat::Png, 8));+ let Some(Item::User { pictures, .. }) = chat.items.last() else { panic!("a sent message") };+ assert_eq!(pictures.len(), 2);+}diff --git a/docs/agent-chat.md b/docs/agent-chat.mdindex 9a03b6908b..c5f14def8f 100644--- a/docs/agent-chat.md+++ b/docs/agent-chat.md@@ -81,8 +81,12 @@ sketch, a brief, a part to fit around.shows a spinner meanwhile, and **Send** waits), and the message tells the agent it was added; afailed import says why and the agent is told nothing of it. Taking a part's chip off the messageleaves the part in the design (Undo removes it).-- Every file is named in the message, and listed under it in the conversation. A file that- can't be attached says why.+- Every file is named in the message. In the conversation the attached pictures show under it as+ thumbnails (hold the pointer over one for a bigger preview, click it to open the viewer, where+ the file's name is shown and ‹ › step through the message's pictures, together with the picture+ of the view); the other files are listed by name. Pictures in any of the formats above are+ decoded in the background, with a spinner until they are in. A file that can't be attached says+ why.Files stay with their message: one queued while the agent works keeps its files, and one sentagain after a sign-in sends them again. When several queued messages go as one prompt, their