SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
Agent chat: PDFs as documents, any other file by reference (#2812) (#2858)
bherbruckdc27988bc283
diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rsindex 014a58428a..d346b1f2d4 100644--- a/apps/solidcraft/src/main.rs+++ b/apps/solidcraft/src/main.rs@@ -179,7 +179,8 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services {}}let all: Vec<&str> = all.iter().map(String::as_str).collect();- let future = file_parent.clone().add_filter(label, &all).pick_file();+ // Any other file attaches by reference, so the list is not the only choice.+ let future = file_parent.clone().add_filter(label, &all).add_filter("All files", &["*"]).pick_file();Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) })})),pick_program: Some(Box::new(move || {diff --git a/assets/locales/en.json b/assets/locales/en.jsonindex 17b92389ba..6c22534563 100644--- a/assets/locales/en.json+++ b/assets/locales/en.json@@ -859,7 +859,7 @@"ui.agent_chat_approval": "Allow the agent to do this?","ui.agent_chat_attach_view": "Attach view","ui.agent_chat_attach_view_tip": "Send a picture of the current view with the message",- "ui.agent_chat_attachable": "Pictures, text and CAD files",+ "ui.agent_chat_attachable": "Pictures, PDFs, text and CAD files","ui.agent_chat_auth_agent": "Sign in with {name}","ui.agent_chat_auth_env": "Set {vars} in your environment and restart SolidCraft.","ui.agent_chat_auth_needed": "The agent asks you to sign in. You sign in with the agent itself; SolidCraft never sees your password or keys.",@@ -891,7 +891,6 @@"ui.agent_chat_file_error": "{name}: {error}","ui.agent_chat_file_heic": "{name}: HEIC pictures can't be read here; export it as JPEG or PNG first","ui.agent_chat_file_importing": "Importing into the design…",- "ui.agent_chat_file_kinds": "{name}: pictures (PNG, JPEG, GIF, WebP, BMP, TIFF), text files (SVG too) and CAD files (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) can be attached","ui.agent_chat_file_loading": "Reading the file…","ui.agent_chat_file_not_a_file": "{name}: not a file","ui.agent_chat_file_not_a_picture": "{name}: not a picture",diff --git a/assets/locales/ja.json b/assets/locales/ja.jsonindex 1a25af5d6e..396aebaee0 100644--- a/assets/locales/ja.json+++ b/assets/locales/ja.json@@ -859,7 +859,7 @@"ui.agent_chat_approval": "エージェントにこの操作を許可しますか?","ui.agent_chat_attach_view": "ビューを添付","ui.agent_chat_attach_view_tip": "現在のビューの画像をメッセージと一緒に送ります",- "ui.agent_chat_attachable": "画像・テキスト・CAD ファイル",+ "ui.agent_chat_attachable": "画像・PDF・テキスト・CAD ファイル","ui.agent_chat_auth_agent": "{name} でサインイン","ui.agent_chat_auth_env": "環境変数 {vars} を設定して SolidCraft を再起動してください。","ui.agent_chat_auth_needed": "エージェントがサインインを求めています。サインインはエージェント自身で行います。SolidCraft がパスワードやキーを見ることはありません。",@@ -891,7 +891,6 @@"ui.agent_chat_file_error": "{name}: {error}","ui.agent_chat_file_heic": "{name}: HEIC 画像はここでは読めません。先に JPEG か PNG に書き出してください","ui.agent_chat_file_importing": "デザインにインポート中…",- "ui.agent_chat_file_kinds": "{name}: 添付できるのは画像 (PNG、JPEG、GIF、WebP、BMP、TIFF)、テキストファイル (SVG も)、CAD ファイル (STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF) です","ui.agent_chat_file_loading": "ファイルを読み込み中…","ui.agent_chat_file_not_a_file": "{name}: ファイルではありません","ui.agent_chat_file_not_a_picture": "{name}: 画像ではありません",diff --git a/assets/locales/zh.json b/assets/locales/zh.jsonindex 6519da022c..c9eafe6ecd 100644--- a/assets/locales/zh.json+++ b/assets/locales/zh.json@@ -859,7 +859,7 @@"ui.agent_chat_approval": "允许智能体执行此操作吗?","ui.agent_chat_attach_view": "附加视图","ui.agent_chat_attach_view_tip": "随消息发送当前视图的图片",- "ui.agent_chat_attachable": "图片、文本和 CAD 文件",+ "ui.agent_chat_attachable": "图片、PDF、文本和 CAD 文件","ui.agent_chat_auth_agent": "使用 {name} 登录","ui.agent_chat_auth_env": "请在环境中设置 {vars},然后重启 SolidCraft。","ui.agent_chat_auth_needed": "智能体要求你登录。登录在智能体本身完成;SolidCraft 从不接触你的密码或密钥。",@@ -891,7 +891,6 @@"ui.agent_chat_file_error": "{name}:{error}","ui.agent_chat_file_heic": "{name}:这里无法读取 HEIC 图片;请先导出为 JPEG 或 PNG","ui.agent_chat_file_importing": "正在导入设计…",- "ui.agent_chat_file_kinds": "{name}:可以附加图片(PNG、JPEG、GIF、WebP、BMP、TIFF)、文本文件(包括 SVG)和 CAD 文件(STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF)","ui.agent_chat_file_loading": "正在读取文件…","ui.agent_chat_file_not_a_file": "{name}:不是文件","ui.agent_chat_file_not_a_picture": "{name}:不是图片",diff --git a/crates/ui-egui/src/agent_chat/acp.rs b/crates/ui-egui/src/agent_chat/acp.rsindex 28315f8d6f..56c159edd4 100644--- a/crates/ui-egui/src/agent_chat/acp.rs+++ b/crates/ui-egui/src/agent_chat/acp.rs@@ -149,6 +149,12 @@ pub fn set_model_params(session: &str, model: &str) -> Value {json!({"sessionId": session, "modelId": model})}+/// Whether the agent takes embedded documents in a message+/// (`agentCapabilities.promptCapabilities.embeddedContext`: it reads `resource` blocks).+pub fn takes_documents(init: &Value) -> bool {+ init.pointer("/agentCapabilities/promptCapabilities/embeddedContext").and_then(Value::as_bool).unwrap_or(false)+}+/// Whether the agent takes pictures in a message (`agentCapabilities.promptCapabilities.image`).pub fn takes_images(init: &Value) -> bool {init.pointer("/agentCapabilities/promptCapabilities/image").and_then(Value::as_bool).unwrap_or(false)@@ -461,6 +467,14 @@ pub fn split_command(line: &str) -> Vec<String> {mod tests {use super::*;+ #[test]+ fn documents_are_taken_only_when_the_agent_says_so() {+ let yes = json!({"agentCapabilities": {"promptCapabilities": {"image": true, "embeddedContext": true}}});+ let no = json!({"agentCapabilities": {"promptCapabilities": {"image": true}}});+ assert!(takes_documents(&yes) && takes_images(&yes));+ assert!(!takes_documents(&no) && !takes_documents(&json!({})));+ }+#[test]fn commands_split_like_a_shell_without_one() {assert_eq!(split_command("gemini --acp"), ["gemini", "--acp"]);diff --git a/crates/ui-egui/src/agent_chat/acp_client.rs b/crates/ui-egui/src/agent_chat/acp_client.rsindex ca9e16b763..82918e90fd 100644--- a/crates/ui-egui/src/agent_chat/acp_client.rs+++ b/crates/ui-egui/src/agent_chat/acp_client.rs@@ -33,6 +33,7 @@ pub struct AcpBackend {session: Option<String>,agent: String,images: bool,+ documents: bool,auth: Vec<acp::AuthMethod>,/// Open permission questions: our key → the agent's request id.permissions: HashMap<u64, Value>,@@ -54,6 +55,7 @@ impl AcpBackend {session: None,agent: String::new(),images: false,+ documents: false,auth: Vec::new(),permissions: HashMap::new(),next_key: 0,@@ -82,6 +84,7 @@ impl AcpBackend {(Pending::Initialize, Ok(v)) => {self.agent = acp::agent_info(&v);self.images = acp::takes_images(&v);+ self.documents = acp::takes_documents(&v);self.auth = acp::auth_methods(&v);self.new_session();}@@ -94,6 +97,7 @@ impl AcpBackend {if let Some((current, available)) = acp::models(&v) {out.push(Event::Models { current, available });}+ out.push(Event::Documents(self.documents));out.push(Event::Ready { agent: self.agent.clone(), images: self.images });}None => out.push(Event::Error("session/new: no sessionId".into())),diff --git a/crates/ui-egui/src/agent_chat/api_client.rs b/crates/ui-egui/src/agent_chat/api_client.rsindex 8c50fd316a..39704695a9 100644--- a/crates/ui-egui/src/agent_chat/api_client.rs+++ b/crates/ui-egui/src/agent_chat/api_client.rs@@ -404,9 +404,11 @@ fn user_message(parts: &[PromptPart]) -> Value {}let content: Vec<Value> = parts.iter()- .map(|p| match p {- PromptPart::Text(t) => json!({"type": "text", "text": t}),- PromptPart::Image { mime, base64 } => json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}}),+ .filter_map(|p| match p {+ PromptPart::Text(t) => Some(json!({"type": "text", "text": t})),+ PromptPart::Image { mime, base64 } => Some(json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}})),+ // This endpoint takes no documents (the chat doesn't send any); the message names the file.+ PromptPart::Document { .. } => None,}).collect();json!({"role": "user", "content": content})diff --git a/crates/ui-egui/src/agent_chat/attach.rs b/crates/ui-egui/src/agent_chat/attach.rsindex 5926e8cae0..926aae003d 100644--- a/crates/ui-egui/src/agent_chat/attach.rs+++ b/crates/ui-egui/src/agent_chat/attach.rs@@ -1,9 +1,10 @@//! Files a person adds to a message (the + button, a drop on the chat, a paste): pictures go to the-//! agent with it, text files go in as text, and CAD files are imported into the design, the agent-//! being told what arrived. Each file is checked and bounded before it is kept; a refused one says-//! why.+//! agent with it, text files go in as text, PDFs go as documents to agents that take them, CAD+//! files are imported into the design, and any other file is named by reference (its type, size+//! and place on this machine) for the agent to read with its own tools. The agent is told what+//! arrived. Each file is checked and bounded before it is kept; a refused one says why.-use std::path::Path;+use std::path::{Path, PathBuf};use std::sync::Arc;use super::backend::{self, PromptPart};@@ -27,6 +28,8 @@ pub const MAX_FILES: usize = 8;const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024;/// Largest text file sent in full.const MAX_TEXT_BYTES: u64 = 256 * 1024;+/// Largest PDF sent as a document; a bigger one goes by reference.+const MAX_PDF_BYTES: u64 = 12 * 1024 * 1024;/// A picture's longer side is scaled down to this before it is sent.pub const MAX_SIDE: u32 = 2048;/// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB,@@ -47,9 +50,9 @@ const TEXT_EXTS: &[&str] = &["svg", "txt", "md", "markdown", "json", "csv", "tsv/// importers themselves).const CAD_EXTS: &[&str] = &["step", "stp", "igs", "iges", "FCStd", "3dm", "stl", "3mf", "obj", "glb", "gltf"];-/// Every extension that can be attached, pictures first (the file picker's list).+/// Every extension the file picker lists first, pictures first (any other file can be chosen too).pub fn extensions() -> Vec<&'static str> {- PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).copied().collect()+ PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).chain(&["pdf"]).copied().collect()}/// What a file is, by its extension.@@ -58,6 +61,8 @@ pub enum Kind {Picture,Text,Cad,+ Pdf,+ /// Anything else, attached by reference.Other,}@@ -66,6 +71,7 @@ pub fn kind(path: &Path) -> Kind {match ext.as_str() {e if PICTURE_EXTS.contains(&e) => Kind::Picture,e if TEXT_EXTS.contains(&e) => Kind::Text,+ "pdf" => Kind::Pdf,_ => {let p = path.to_string_lossy();if solidcraft_engine::io::is_step_path(&p) || solidcraft_engine::io::is_mesh_path(&p) { Kind::Cad } else { Kind::Other }@@ -82,6 +88,17 @@ pub enum Content {size: [u32; 2],},Text(Arc<String>),+ /// A PDF, sent whole as a document to an agent that takes them.+ Pdf {+ bytes: Arc<Vec<u8>>,+ path: PathBuf,+ },+ /// A file the agent is pointed to, not given: what it is, its size and where it is.+ Reference {+ path: PathBuf,+ size: u64,+ mime: &'static str,+ },/// A CAD file imported into the design as its own component.Imported,}@@ -104,6 +121,70 @@ impl Attachment {pub fn is_picture(&self) -> bool {matches!(self.content, Content::Picture { .. })}+ /// The chip's mark for a file with no thumbnail: its extension, in capitals, cut to fit.+ pub fn mark(&self) -> String {+ let ext = Path::new(&self.name).extension().and_then(|e| e.to_str()).unwrap_or("");+ ext.chars().take(4).collect::<String>().to_uppercase()+ }+}++/// What a file is called to the agent, by its extension.+fn mime_of(path: &Path) -> &'static str {+ let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default();+ match ext.as_str() {+ "pdf" => "application/pdf",+ "zip" => "application/zip",+ "7z" => "application/x-7z-compressed",+ "gz" | "tgz" => "application/gzip",+ "dxf" => "image/vnd.dxf",+ "dwg" => "image/vnd.dwg",+ "step" | "stp" => "model/step",+ "iges" | "igs" => "model/iges",+ "stl" => "model/stl",+ "obj" => "model/obj",+ "3mf" => "model/3mf",+ "gltf" => "model/gltf+json",+ "glb" => "model/gltf-binary",+ "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",+ "mp4" | "mov" | "webm" => "video/mp4",+ "wav" => "audio/wav",+ "mp3" => "audio/mpeg",+ _ => "application/octet-stream",+ }+}++/// A size for people: `1.5 MB`.+fn size_text(bytes: u64) -> String {+ const KB: f64 = 1024.0;+ let b = bytes as f64;+ if b < KB {+ format!("{bytes} bytes")+ } else if b < KB * KB {+ format!("{:.0} KB", b / KB)+ } else if b < KB * KB * KB {+ format!("{:.1} MB", b / KB / KB)+ } else {+ format!("{:.1} GB", b / KB / KB / KB)+ }+}++/// A `file://` link to a path, for a document sent to an agent.+fn file_uri(path: &Path) -> String {+ let mut s = path.to_string_lossy().replace('\\', "/");+ if !s.starts_with('/') {+ s.insert(0, '/');+ }+ let mut out = String::from("file://");+ for b in s.bytes() {+ if b.is_ascii_alphanumeric() || matches!(b, b'/' | b'-' | b'_' | b'.' | b'~' | b':') {+ out.push(char::from(b));+ } else {+ out.push_str(&format!("%{b:02X}"));+ }+ }+ out}pub(crate) fn name_of(path: &Path) -> String {@@ -134,13 +215,29 @@ pub fn read(path: &Path) -> Result<Attachment, String> {Ok(Attachment { id: 0, name, content: Content::Text(Arc::new(text)), thumb: None })}Kind::Cad => Err(refused("ui.agent_chat_file_cad_not_read", &name, &[])),+ Kind::Pdf => {+ let path = absolute(path);+ // Too big to send: the agent is pointed to it instead.+ if meta.len() > MAX_PDF_BYTES {+ return Ok(Attachment { id: 0, name, content: Content::Reference { path, size: meta.len(), mime: "application/pdf" }, thumb: None });+ }+ let bytes = std::fs::read(&path).map_err(|e| failed(&name, e))?;+ Ok(Attachment { id: 0, name, content: Content::Pdf { bytes: Arc::new(bytes), path }, thumb: None })+ }Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => {Err(refused("ui.agent_chat_file_heic", &name, &[]))}- Kind::Other => Err(refused("ui.agent_chat_file_kinds", &name, &[])),+ Kind::Other => {+ Ok(Attachment { id: 0, name, content: Content::Reference { path: absolute(path), size: meta.len(), mime: mime_of(path) }, thumb: None })+ }}}+/// The path as an absolute one, so the agent (working in another folder) finds the file.+fn absolute(path: &Path) -> PathBuf {+ std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf())+}+/// A picture from a file's bytes: decoded within limits, and sent as it came when the agents read/// that format (PNG, JPEG, GIF, WebP), it is at most [`MAX_SIDE`] on a side and at most/// [`MAX_SENT_BYTES`]; otherwise re-encoded (see [`encoded`]).@@ -239,10 +336,10 @@ fn thumbnail(img: &image::DynamicImage) -> Option<Arc<egui::ColorImage>> {/// What the message says about its files, after the person's words: the text files' text, and the/// pictures and imported parts by name (the agent sees the pictures themselves as images).-/// `pictures_sent`: whether this agent takes pictures. The text files share `budget` characters+/// `pictures_sent` and `documents_sent`: whether this agent takes pictures and PDFs. The text files share `budget` characters/// in order: one that doesn't fit is cut and says how much is missing, and one past the budget/// is named only, so the message stays within the prompt cap.-pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String {+pub fn note(files: &[Attachment], pictures_sent: bool, documents_sent: bool, budget: usize) -> String {let mut out = String::new();let pictures: Vec<&str> = files.iter().filter(|f| f.is_picture()).map(|f| f.name.as_str()).collect();if !pictures.is_empty() {@@ -270,6 +367,31 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> Stringleft = 0;}}+ Content::Pdf { path, bytes } if documents_sent => {+ out += &format!(+ "\n\nAttached PDF `{}` ({}, at {}): sent with this message as a document.",+ f.name,+ size_text(bytes.len() as u64),+ path.display()+ );+ }+ // An agent that takes no documents is pointed to the file, as for any other file.+ Content::Pdf { path, bytes } => {+ out += &format!(+ "\n\nAttached file `{}` (PDF, {}, at {}): read it with your own tools; it is not in this message.",+ f.name,+ size_text(bytes.len() as u64),+ path.display()+ );+ }+ Content::Reference { path, size, mime } => {+ out += &format!(+ "\n\nAttached file `{}` ({mime}, {}, at {}): read it with your own tools; it is not in this message.",+ f.name,+ size_text(*size),+ path.display()+ );+ }Content::Imported => out += &format!("\n\n(I added `{}` to the design as a component of its own.)", f.name),Content::Picture { .. } => {}}@@ -277,12 +399,18 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> Stringout}-/// The pictures as image parts of the message.-pub fn parts(files: &[Attachment]) -> Vec<PromptPart> {+/// The pictures and PDFs as parts of the message, to the agents that take them.+pub fn parts(files: &[Attachment], pictures: bool, documents: bool) -> Vec<PromptPart> {files.iter().filter_map(|f| match &f.content {- Content::Picture { mime, bytes, .. } => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),+ Content::Picture { mime, bytes, .. } if pictures => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),+ Content::Pdf { bytes, path } if documents => Some(PromptPart::Document {+ name: f.name.clone(),+ uri: file_uri(path),+ mime: "application/pdf".to_string(),+ base64: backend::base64(bytes),+ }),_ => None,}).collect()@@ -390,9 +518,6 @@ mod tests {let binary = d.join("bin.json");std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();assert!(read(&binary).unwrap_err().contains("UTF-8"));- let other = d.join("movie.mov");- std::fs::write(&other, b"x").unwrap();- assert!(read(&other).unwrap_err().contains("can be attached"));assert!(read(&d.join("missing.png")).is_err());assert_eq!(kind(Path::new("part.STEP")), Kind::Cad);assert_eq!(kind(Path::new("mount.stl")), Kind::Cad);@@ -411,12 +536,9 @@ mod tests {let d = dir("lang");let binary = d.join("bin.json");std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();- let other = d.join("movie.mov");- std::fs::write(&other, b"x").unwrap();{let _ja = crate::i18n::scope("ja");assert!(read(&binary).unwrap_err().contains("UTF-8 のテキストではありません"));- assert!(read(&other).unwrap_err().starts_with("movie.mov: 添付できるのは"));}{let _zh = crate::i18n::scope("zh");@@ -467,12 +589,12 @@ mod tests {fn attached_text_is_kept_within_the_prompt_budget() {let file = |n: &str, len: usize| Attachment { id: 0, name: n.into(), content: Content::Text(Arc::new("x".repeat(len))), thumb: None };let files = vec![file("a.md", 600), file("b.md", 600), file("c.md", 600)];- let n = note(&files, true, 1000);+ let n = note(&files, true, false, 1000);assert!(n.contains("`a.md`") && n.contains("`b.md`") && n.contains("`c.md`"), "every file is named: {n}");assert!(n.matches('x').count() <= 1000, "the files' text stays within the budget");assert!(n.contains("200 more characters"), "a cut file says how much is missing: {n}");assert!(n.contains("left out"), "a file past the budget is named only: {n}");- assert_eq!(note(&files, true, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");+ assert_eq!(note(&files, true, false, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");}#[test]@@ -481,13 +603,77 @@ mod tests {let text = Attachment { id: 0, name: "brief.md".into(), content: Content::Text(Arc::new("wall 2 mm\n".into())), thumb: None };let part = Attachment::imported("motor.step".into());let files = vec![pic, text, part];- let n = note(&files, true, usize::MAX);+ let n = note(&files, true, false, usize::MAX);assert!(n.contains("Attached pictures: target.png."), "{n}");assert!(n.contains("Attached file `brief.md`:\n```\nwall 2 mm\n```"), "{n}");assert!(n.contains("`motor.step` to the design"), "{n}");- assert!(note(&files, false, usize::MAX).contains("not sent"));- let p = parts(&files);+ assert!(note(&files, false, false, usize::MAX).contains("not sent"));+ let p = parts(&files, true, true);assert_eq!(p.len(), 1);assert!(matches!(&p[0], PromptPart::Image { mime, base64 } if mime == "image/png" && base64.starts_with("iVBORw0KGgo")));}++ /// Any file that is neither a picture, text, CAD nor a PDF is named by reference (what it is,+ /// its size and its place) instead of refused; the agent reads it with its own tools.+ #[test]+ fn other_files_go_by_reference() {+ let d = dir("reference");+ let zip = d.join("fixtures.zip");+ std::fs::write(&zip, vec![0u8; 3000]).unwrap();+ let dxf = d.join("outline.DXF");+ std::fs::write(&dxf, b"0\nSECTION\n").unwrap();+ let a = read(&zip).unwrap();+ let Content::Reference { path, size, mime } = &a.content else { panic!("a reference") };+ assert_eq!((*size, *mime), (3000, "application/zip"));+ assert!(path.is_absolute() && path.ends_with("fixtures.zip"));+ assert!(a.thumb.is_none() && a.mark() == "ZIP");+ assert!(matches!(read(&dxf).unwrap().content, Content::Reference { mime: "image/vnd.dxf", .. }), "by extension, in any case");+ let bare = d.join("LICENSE_BLOB");+ std::fs::write(&bare, b"x").unwrap();+ assert!(matches!(read(&bare).unwrap().content, Content::Reference { mime: "application/octet-stream", .. }), "no extension");+ let files = vec![a];+ let n = note(&files, true, true, usize::MAX);+ assert!(n.contains("Attached file `fixtures.zip` (application/zip, 3 KB, at "), "{n}");+ assert!(n.contains("fixtures.zip): read it with your own tools"), "{n}");+ assert!(parts(&files, true, true).is_empty(), "a reference sends nothing but the note");+ assert!(read(&d).unwrap_err().contains("not a file"), "a folder is refused");+ assert!(read(&d.join("missing.bin")).is_err());+ let _ = std::fs::remove_dir_all(&d);+ }++ /// A PDF goes whole as a document to an agent that takes them (and is named with its place);+ /// to one that doesn't, and when it is too big to send, the agent is pointed to the file.+ #[test]+ fn pdfs_go_as_documents_or_by_reference() {+ let d = dir("pdf");+ let pdf = d.join("brief 1.pdf");+ std::fs::write(&pdf, b"%PDF-1.4\n%%EOF\n").unwrap();+ let a = read(&pdf).unwrap();+ assert!(matches!(&a.content, Content::Pdf { bytes, .. } if bytes.starts_with(b"%PDF")));+ assert_eq!(a.mark(), "PDF");+ let files = vec![a];+ let sent = parts(&files, false, true);+ let [PromptPart::Document { name, uri, mime, base64 }] = sent.as_slice() else { panic!("one document: {sent:?}") };+ assert_eq!((name.as_str(), mime.as_str()), ("brief 1.pdf", "application/pdf"));+ assert!(uri.starts_with("file://") && uri.ends_with("/brief%201.pdf"), "{uri}");+ assert!(base64.starts_with("JVBERi0"), "the PDF's bytes: {base64}");+ assert!(note(&files, false, true, usize::MAX).contains("sent with this message as a document"));+ assert!(parts(&files, true, false).is_empty(), "an agent with no documents gets none");+ let n = note(&files, true, false, usize::MAX);+ assert!(n.contains("(PDF, 15 bytes, at ") && n.contains("read it with your own tools"), "{n}");+ let big = d.join("drawing set.pdf");+ std::fs::write(&big, vec![b'x'; usize::try_from(MAX_PDF_BYTES).unwrap() + 1]).unwrap();+ let b = read(&big).unwrap();+ assert!(matches!(b.content, Content::Reference { mime: "application/pdf", .. }), "too big to send: by reference");+ assert!(parts(&[b], true, true).is_empty());+ let _ = std::fs::remove_dir_all(&d);+ }++ #[test]+ fn links_to_files_are_escaped() {+ assert_eq!(file_uri(Path::new("/home/a b/ü.pdf")), "file:///home/a%20b/%C3%BC.pdf");+ assert_eq!(file_uri(Path::new("C:\\Users\\a\\x.pdf")), "file:///C:/Users/a/x.pdf");+ assert_eq!(size_text(1536), "2 KB");+ assert_eq!(size_text(5 * 1024 * 1024 + 512 * 1024), "5.5 MB");+ }}diff --git a/crates/ui-egui/src/agent_chat/backend.rs b/crates/ui-egui/src/agent_chat/backend.rsindex a44a54826a..a470ccad93 100644--- a/crates/ui-egui/src/agent_chat/backend.rs+++ b/crates/ui-egui/src/agent_chat/backend.rs@@ -19,11 +19,21 @@ pub enum PromptPart {mime: String,base64: String,},+ /// A file the agent reads as a document (a PDF), sent only to agents that take documents.+ Document {+ name: String,+ /// Where it is on this machine (a `file://` link).+ uri: String,+ mime: String,+ base64: String,+ },}/// What a backend reports to the panel.#[derive(Clone, Debug, PartialEq)]pub enum Event {+ /// Whether the agent takes documents (PDFs) in a message; sent before [`Event::Ready`].+ Documents(bool),/// Ready for a message. `agent`: its name and version; `images`: it takes pictures.Ready { agent: String, images: bool },/// Text it streams (`thought`: its reasoning, shown dimmed).@@ -136,6 +146,9 @@ pub fn content_blocks(parts: &[PromptPart]) -> Vec<Value> {.map(|p| match p {PromptPart::Text(t) => serde_json::json!({"type": "text", "text": t}),PromptPart::Image { mime, base64 } => serde_json::json!({"type": "image", "mimeType": mime, "data": base64}),+ PromptPart::Document { uri, mime, base64, .. } => {+ serde_json::json!({"type": "resource", "resource": {"uri": uri, "mimeType": mime, "blob": base64}})+ }}).collect()}@@ -158,4 +171,19 @@ mod tests {assert_eq!(b[1]["type"], "image");assert_eq!(b[1]["mimeType"], "image/png");}++ #[test]+ fn documents_go_as_embedded_resources() {+ let doc = PromptPart::Document {+ name: "brief.pdf".into(),+ uri: "file:///a/brief.pdf".into(),+ mime: "application/pdf".into(),+ base64: "AA==".into(),+ };+ let b = content_blocks(&[doc]);+ assert_eq!(b[0]["type"], "resource");+ assert_eq!(b[0]["resource"]["uri"], "file:///a/brief.pdf");+ assert_eq!(b[0]["resource"]["mimeType"], "application/pdf");+ assert_eq!(b[0]["resource"]["blob"], "AA==");+ }}diff --git a/crates/ui-egui/src/agent_chat/mod.rs b/crates/ui-egui/src/agent_chat/mod.rsindex 42b0ecca4b..4dad5c60c3 100644--- a/crates/ui-egui/src/agent_chat/mod.rs+++ b/crates/ui-egui/src/agent_chat/mod.rs@@ -303,6 +303,8 @@ pub struct AgentChat {state: Option<State>,agent: String,images: bool,+ /// The agent takes documents (PDFs) in a message.+ documents: bool,/// Send a picture of the view with the next message.pub attach_view: bool,auth: Vec<acp::AuthMethod>,@@ -512,6 +514,7 @@ impl AgentChat {/// Take in what the backend reported.fn on_event(&mut self, e: Event) -> Option<Outcome> {match e {+ Event::Documents(documents) => self.documents = documents,Event::Ready { agent, images } => {self.agent = agent;self.images = images;@@ -960,14 +963,14 @@ fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option<usize>,/// files are named after the text (text files in full), and pictures go as images to an agent/// that takes them. `at` is the message's place in the conversation, where its picture is kept.fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>, note: Option<&str>) {- let pictures = app.agent_chat.images;+ let (pictures, documents) = (app.agent_chat.images, app.agent_chat.documents);let lead = match note {Some(n) => format!("{n}\n\n{text}"),None => text.clone(),};// The whole message stays within the cap: the files' text gets what the words leave.let budget = MAX_PROMPT.saturating_sub(lead.chars().count() + FILE_NAMES_ROOM);- let about_files = attach::note(&files, pictures, budget);+ let about_files = attach::note(&files, pictures, documents, budget);let sent: String = format!("{lead}{about_files}").chars().take(MAX_PROMPT).collect();let mut parts = vec![PromptPart::Text(sent)];if image {@@ -979,9 +982,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usiErr(e) => log::warn!("the view's picture is left out of the prompt: {e}"),}}- if pictures {- parts.extend(attach::parts(&files));- }+ parts.extend(attach::parts(&files, pictures, documents));// The undo step is named after the message, or its first file when it has no words.let first_line = text.lines().find(|l| !l.trim().is_empty()).or_else(|| files.first().map(|f| f.name.as_str())).unwrap_or("");let label: String = first_line.chars().take(48).collect();@@ -1017,9 +1018,10 @@ pub fn attach_path(app: &mut SolveApp, path: &str) -> std::result::Result<Stringcrate::file_load::begin_request(app, request).map_err(|e| trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &e)]))?;return Ok(app.agent_chat.importing(name, rx));} else {- // A picture is decoded off the window thread: a large one takes long enough to stop it.+ // A picture is decoded and a PDF read off the window thread: a large one takes long enough+ // to stop it.#[cfg(not(target_arch = "wasm32"))]- if attach::kind(p) == attach::Kind::Picture {+ if matches!(attach::kind(p), attach::Kind::Picture | attach::Kind::Pdf) {// A missing file is said at once; reading and decoding wait for the worker.if !p.is_file() {return Err(trf("ui.agent_chat_file_not_a_file", &[("name", &attach::name_of(p))]));@@ -1654,6 +1656,7 @@ pub fn status(app: &SolveApp) -> Value {"state": c.state().id(),"agent": c.agent,"images": c.images,+ "documents": c.documents,"authStatus": c.auth_status,"authMethods": c.auth.iter().map(|m| json!({"id": m.id, "name": m.name, "kind": match &m.kind { acp::AuthKind::Agent => "agent", acp::AuthKind::Terminal { .. } => "terminal", acp::AuthKind::EnvVar { .. } => "env_var" }})).collect::<Vec<_>>(),"toolsPort": c.tools_port,diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rsindex 7ad6cd836e..354185a2dc 100644--- a/crates/ui-egui/src/agent_chat/panel.rs+++ b/crates/ui-egui/src/agent_chat/panel.rs@@ -1278,8 +1278,11 @@ fn chip(ui: &mut egui::Ui, thumbs: &mut std::collections::HashMap<u64, egui::Texui.image((tex.id(), size * (22.0 / size.x.max(size.y).max(1.0)))).rect.width()}None => {- let (r, _) = ui.allocate_exact_size(vec2(16.0, 20.0), Sense::hover());- ui.painter().rect_stroke(r.shrink2(vec2(2.0, 3.0)), 2.0, Stroke::new(1.2, t.text_dim), egui::StrokeKind::Inside);+ // A page marked with the file's type.+ let (r, _) = ui.allocate_exact_size(vec2(28.0, 20.0), Sense::hover());+ let color = if matches!(f.content, attach::Content::Pdf { .. }) { t.error } else { t.text_dim };+ ui.painter().rect_stroke(r.shrink2(vec2(1.0, 1.0)), 2.0, Stroke::new(1.2, color), egui::StrokeKind::Inside);+ ui.painter().text(r.center(), Align2::CENTER_CENTER, f.mark(), egui::FontId::proportional(8.5), color);r.width()}};