Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
gpui: Support image paste and async clipboard reads on web (#62871)
Gaauwe Romboutsd70c45e5bb1d
diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rsindex 72833abe527..dedebbe5857 100644--- a/crates/gpui/src/app.rs+++ b/crates/gpui/src/app.rs@@ -44,10 +44,10 @@ pub use visual_test_context::*;use crate::InspectorElementRegistry;use crate::{Action, ActionBuildError, ActionRegistry, Any, AnyView, AnyWindowHandle, AppContext, Arena,- ArenaBox, Asset, AssetSource, BackgroundExecutor, Bounds, ClipboardItem, CursorStyle,- DispatchPhase, DisplayId, EventEmitter, ExternalDragPayload, FocusHandle, FocusMap,- ForegroundExecutor, Global, KeyBinding, KeyContext, Keymap, Keystroke, LayoutId, Menu,- MenuItem, OwnedMenu, PathPromptOptions, Pixels, Platform, PlatformDisplay,+ ArenaBox, Asset, AssetSource, BackgroundExecutor, Bounds, ClipboardItem, ClipboardReadError,+ CursorStyle, DispatchPhase, DisplayId, EventEmitter, ExternalDragPayload, FocusHandle,+ FocusMap, ForegroundExecutor, Global, KeyBinding, KeyContext, Keymap, Keystroke, LayoutId,+ Menu, MenuItem, OwnedMenu, PathPromptOptions, Pixels, Platform, PlatformDisplay,PlatformKeyboardLayout, PlatformKeyboardMapper, Point, Priority, PromptBuilder, PromptButton,PromptHandle, PromptLevel, Render, RenderImage, RenderablePromptHandle, Reservation,ScreenCaptureSource, SharedString, SubscriberSet, Subscription, SvgRenderer,@@ -1381,6 +1381,19 @@ impl App {self.platform.read_from_clipboard()}+ /// Reads data from the platform clipboard, resolving once the contents+ /// are available.+ ///+ /// Prefer this over [`App::read_from_clipboard`] in code that can await:+ /// on platforms where clipboard access is asynchronous and+ /// permission-gated (e.g. web), the synchronous read always returns+ /// `None` while this method performs a real read.+ pub fn read_from_clipboard_async(+ &self,+ ) -> Task<Result<Option<ClipboardItem>, ClipboardReadError>> {+ self.platform.read_from_clipboard_async()+ }+/// Sets the text rendering mode for the application.pub fn set_text_rendering_mode(&mut self, mode: TextRenderingMode) {self.text_rendering_mode.set(mode);diff --git a/crates/gpui/src/input.rs b/crates/gpui/src/input.rsindex 4e204e97880..6de5719c699 100644--- a/crates/gpui/src/input.rs+++ b/crates/gpui/src/input.rs@@ -1,4 +1,6 @@-use crate::{App, Bounds, Context, Entity, InputHandler, Pixels, UTF16Selection, Window};+use crate::{+ App, Bounds, ClipboardItem, Context, Entity, InputHandler, Pixels, UTF16Selection, Window,+};use std::ops::Range;/// Implement this trait to allow views to handle textual input when implementing an editor, field, etc.@@ -35,6 +37,13 @@ pub trait EntityInputHandler: 'static + Sized {/// See [`InputHandler::unmark_text`] for detailsfn unmark_text(&mut self, window: &mut Window, cx: &mut Context<Self>);+ /// See [`InputHandler::paste`] for details+ fn paste(&mut self, item: ClipboardItem, window: &mut Window, cx: &mut Context<Self>) {+ if let Some(text) = item.text() {+ self.replace_text_in_range(None, &text, window, cx);+ }+ }+/// See [`InputHandler::replace_text_in_range`] for detailsfn replace_text_in_range(&mut self,@@ -179,6 +188,11 @@ impl<V: EntityInputHandler> InputHandler for ElementInputHandler<V> {.update(cx, |view, cx| view.unmark_text(window, cx));}+ fn paste(&mut self, item: ClipboardItem, window: &mut Window, cx: &mut App) {+ self.view+ .update(cx, |view, cx| view.paste(item, window, cx));+ }+fn bounds_for_range(&mut self,range_utf16: Range<usize>,diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rsindex 14a0761910e..74457652938 100644--- a/crates/gpui/src/platform.rs+++ b/crates/gpui/src/platform.rs@@ -309,6 +309,17 @@ pub trait Platform: 'static {fn read_from_clipboard(&self) -> Option<ClipboardItem>;fn write_to_clipboard(&self, item: ClipboardItem);+ /// Reads the clipboard, resolving once its contents are available.+ ///+ /// Most platforms read synchronously and return a ready task. Platforms+ /// whose clipboard access is inherently asynchronous and permission-gated+ /// (e.g. the browser's async clipboard API) override this method; on those+ /// platforms [`Platform::read_from_clipboard`] cannot return the clipboard+ /// contents, so callers that can await should prefer this method.+ fn read_from_clipboard_async(&self) -> Task<Result<Option<ClipboardItem>, ClipboardReadError>> {+ Task::ready(Ok(self.read_from_clipboard()))+ }+#[cfg(any(target_os = "linux", target_os = "freebsd"))]fn read_from_primary(&self) -> Option<ClipboardItem>;#[cfg(any(target_os = "linux", target_os = "freebsd"))]@@ -1507,6 +1518,12 @@ impl PlatformInputHandler {.ok();}+ pub fn paste(&mut self, item: ClipboardItem) {+ self.cx+ .update(|window, cx| self.handler.paste(item, window, cx))+ .ok();+ }+pub fn bounds_for_range(&mut self, range_utf16: Range<usize>) -> Option<Bounds<Pixels>> {self.cx.update(|window, cx| self.handler.bounds_for_range(range_utf16, window, cx))@@ -1713,6 +1730,18 @@ pub trait InputHandler: 'static {/// Corresponds to [unmarkText()](https://developer.apple.com/documentation/appkit/nstextinputclient/1438239-unmarktext)fn unmark_text(&mut self, window: &mut Window, cx: &mut App);+ /// Insert a platform-initiated paste at the current selection.+ ///+ /// Platforms that deliver paste as an input event rather than through an+ /// application-defined action (e.g. the DOM `paste` event on web) call+ /// this with the full clipboard contents. The default implementation+ /// inserts only the plain-text portion of the item.+ fn paste(&mut self, item: ClipboardItem, window: &mut Window, cx: &mut App) {+ if let Some(text) = item.text() {+ self.replace_text_in_range(None, &text, window, cx);+ }+ }+/// Get the bounds of the given document range in screen coordinates/// Corresponds to [firstRect(forCharacterRange:actualRange:)](https://developer.apple.com/documentation/appkit/nstextinputclient/1438240-firstrect)///@@ -2278,6 +2307,40 @@ pub struct ClipboardItem {pub entries: Vec<ClipboardEntry>,}+/// An error produced by [`Platform::read_from_clipboard_async`].+///+/// Callers surface these failures to users, so the variants distinguish+/// conditions that call for different user-facing guidance.+#[derive(Clone, Debug, PartialEq, Eq)]+pub enum ClipboardReadError {+ /// The platform clipboard is not available in this context, e.g. the+ /// browser does not expose the async clipboard API or the page is not a+ /// secure context.+ Unavailable,+ /// The platform refused access, e.g. the user declined the browser's+ /// clipboard permission prompt or paste confirmation.+ Denied(String),+ /// The clipboard contents could not be converted into a+ /// [`ClipboardItem`].+ UnsupportedContent,+}++impl std::fmt::Display for ClipboardReadError {+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::Unavailable => formatter.write_str("the clipboard is unavailable"),+ Self::Denied(message) => {+ write!(formatter, "clipboard access was denied: {message}")+ }+ Self::UnsupportedContent => {+ formatter.write_str("the clipboard contents are unsupported")+ }+ }+ }+}++impl std::error::Error for ClipboardReadError {}+/// Either a ClipboardString or a ClipboardImage#[derive(Clone, Debug, Eq, PartialEq)]pub enum ClipboardEntry {diff --git a/crates/gpui_web/Cargo.toml b/crates/gpui_web/Cargo.tomlindex aed7c9fd274..c02b57642ee 100644--- a/crates/gpui_web/Cargo.toml+++ b/crates/gpui_web/Cargo.toml@@ -39,16 +39,21 @@ wasm_thread = { git = "https://github.com/zed-industries/wasm_thread", rev = "0c], optional = true }web-sys = { version = "0.3", features = ["console",+ "Blob","Clipboard","ClipboardEvent",+ "ClipboardItem","CompositionEvent","CssStyleDeclaration","DataTransfer",+ "DataTransferItem",+ "DataTransferItemList","Document","DomRect","DragEvent","Element","EventTarget",+ "File","HtmlCanvasElement","HtmlElement","HtmlInputElement",diff --git a/crates/gpui_web/src/events.rs b/crates/gpui_web/src/events.rsindex 471732671e1..74ad1b69192 100644--- a/crates/gpui_web/src/events.rs+++ b/crates/gpui_web/src/events.rs@@ -1,10 +1,10 @@use std::rc::Rc;use gpui::{- Capslock, DispatchEventResult, KeyDownEvent, KeyUpEvent, Keystroke, Modifiers,- ModifiersChangedEvent, MouseButton, MouseDownEvent, MouseExitEvent, MouseMoveEvent,- MouseUpEvent, NavigationDirection, Pixels, PlatformInput, Point, ScrollDelta, ScrollWheelEvent,- TouchPhase, point, px,+ Capslock, ClipboardEntry, ClipboardItem, ClipboardString, DispatchEventResult, Image,+ ImageFormat, KeyDownEvent, KeyUpEvent, Keystroke, Modifiers, ModifiersChangedEvent,+ MouseButton, MouseDownEvent, MouseExitEvent, MouseMoveEvent, MouseUpEvent, NavigationDirection,+ Pixels, PlatformInput, Point, ScrollDelta, ScrollWheelEvent, TouchPhase, point, px,};use wasm_bindgen::prelude::*;@@ -469,6 +469,12 @@ impl WebWindowInner {/// read API cannot fit that synchronous signature, while `ClipboardEvent`/// exposes `clipboardData` synchronously inside the event. It fires for/// any browser-initiated paste (keyboard, menu bar, context menu).+ ///+ /// Text-only pastes reach the input handler synchronously. Pasted image+ /// files only expose their bytes through asynchronous blob reads, so+ /// pastes containing images are delivered once those reads resolve — to+ /// whichever input handler is focused at that point, matching how an+ /// application-level paste action would behave.fn register_paste(self: &Rc<Self>) -> EventListenerHandle {let this = Rc::clone(self);self.listen_input("paste", move |event: JsValue| {@@ -476,16 +482,70 @@ impl WebWindowInner {let Some(clipboard_data) = event.clipboard_data() else {return;};- let Ok(text) = clipboard_data.get_data("text/plain") else {+ let text = clipboard_data+ .get_data("text/plain")+ .ok()+ .filter(|text| !text.is_empty());++ // File handles must be collected synchronously: the browser+ // clears `clipboardData`'s item list once this handler returns,+ // while the `File`s themselves stay readable afterwards.+ let mut image_files = Vec::new();+ let items = clipboard_data.items();+ for index in 0..items.length() {+ let Some(item) = items.get(index) else {+ continue;+ };+ if item.kind() != "file" {+ continue;+ }+ let Some(format) = ImageFormat::from_mime_type(&item.type_()) else {+ continue;+ };+ if let Ok(Some(file)) = item.get_as_file() {+ image_files.push((format, file));+ }+ }++ if text.is_none() && image_files.is_empty() {return;- };- if text.is_empty() {+ }+ event.prevent_default();++ if image_files.is_empty() {+ if let Some(text) = text {+ this.with_input_handler(|handler| {+ handler.paste(ClipboardItem::new_string(text));+ });+ }return;}- event.prevent_default();- this.with_input_handler(|handler| {- handler.replace_text_in_range(None, &text);+ let this = Rc::clone(&this);+ wasm_bindgen_futures::spawn_local(async move {+ let mut entries = Vec::new();+ if let Some(text) = text {+ entries.push(ClipboardEntry::String(ClipboardString::new(text)));+ }+ for (format, file) in image_files {+ match crate::platform::read_blob_bytes(&file).await {+ Ok(bytes) => {+ entries.push(ClipboardEntry::Image(Image::from_bytes(format, bytes)));+ }+ Err(error) => {+ log::error!(+ "failed to read pasted image: {}",+ crate::platform::js_error_message(&error)+ );+ }+ }+ }+ if entries.is_empty() {+ return;+ }+ this.with_input_handler(|handler| {+ handler.paste(ClipboardItem { entries });+ });});})}diff --git a/crates/gpui_web/src/platform.rs b/crates/gpui_web/src/platform.rsindex ecf2736ceee..65e092cac8a 100644--- a/crates/gpui_web/src/platform.rs+++ b/crates/gpui_web/src/platform.rs@@ -7,10 +7,11 @@ use crate::window::WebWindow;use anyhow::Result;use futures::channel::oneshot;use gpui::{- Action, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DummyKeyboardMapper,- ForegroundExecutor, Keymap, Menu, MenuItem, PathPromptOptions, Platform, PlatformDisplay,- PlatformKeyboardLayout, PlatformKeyboardMapper, PlatformTextSystem, PlatformWindow, Task,- ThermalState, WindowAppearance, WindowKind, WindowParams, popup::PopupNotSupportedError,+ Action, AnyWindowHandle, BackgroundExecutor, ClipboardEntry, ClipboardItem, ClipboardReadError,+ ClipboardString, CursorStyle, DummyKeyboardMapper, ForegroundExecutor, Image, ImageFormat,+ Keymap, Menu, MenuItem, PathPromptOptions, Platform, PlatformDisplay, PlatformKeyboardLayout,+ PlatformKeyboardMapper, PlatformTextSystem, PlatformWindow, Task, ThermalState,+ WindowAppearance, WindowKind, WindowParams, popup::PopupNotSupportedError,};use gpui_wgpu::{PreparedWebGraphics, WebBackendPreference, WgpuContext, wgpu};use std::{@@ -555,6 +556,67 @@ impl Platform for WebPlatform {None}+ fn read_from_clipboard_async(&self) -> Task<Result<Option<ClipboardItem>, ClipboardReadError>> {+ let navigator = self.browser_window.navigator();+ // `navigator.clipboard` is undefined outside secure contexts; probing+ // first avoids a wasm-bindgen abort from invoking a method on+ // `undefined`.+ let clipboard_available = js_sys::Reflect::get(navigator.as_ref(), &"clipboard".into())+ .is_ok_and(|clipboard| !clipboard.is_undefined() && !clipboard.is_null());+ if !clipboard_available {+ return Task::ready(Err(ClipboardReadError::Unavailable));+ }+ // `read()` must be called synchronously so it is still covered by the+ // user activation (e.g. the click on a context-menu item) that+ // triggered this read.+ let read = navigator.clipboard().read();+ self.foreground_executor.spawn(async move {+ let items = wasm_bindgen_futures::JsFuture::from(read)+ .await+ .map_err(clipboard_read_rejection_error)?;+ let mut entries = Vec::new();+ let mut saw_unsupported_type = false;+ for item in js_sys::Array::from(&items).iter() {+ let item: web_sys::ClipboardItem = item.unchecked_into();+ for mime_type in item.types().iter() {+ let Some(mime_type) = mime_type.as_string() else {+ continue;+ };+ // Only fetch blobs for types we can convert; copies from+ // other web apps routinely carry `text/html` and+ // `web application/...` custom formats whose `getType`+ // fetches would be wasted or rejected.+ if mime_type == "text/plain" {+ match read_clipboard_item_text(&item, &mime_type).await {+ Ok(text) if !text.is_empty() => {+ entries.push(ClipboardEntry::String(ClipboardString::new(text)));+ }+ Ok(_) => {}+ Err(error) => log_clipboard_entry_error(&mime_type, &error),+ }+ } else if let Some(format) = ImageFormat::from_mime_type(&mime_type) {+ match read_clipboard_item_bytes(&item, &mime_type).await {+ Ok(bytes) => {+ entries+ .push(ClipboardEntry::Image(Image::from_bytes(format, bytes)));+ }+ Err(error) => log_clipboard_entry_error(&mime_type, &error),+ }+ } else {+ saw_unsupported_type = true;+ }+ }+ }+ if !entries.is_empty() {+ Ok(Some(ClipboardItem { entries }))+ } else if saw_unsupported_type {+ Err(ClipboardReadError::UnsupportedContent)+ } else {+ Ok(None)+ }+ })+ }+fn write_to_clipboard(&self, item: ClipboardItem) {if let Some(text) = item.text()&& let Some(window) = web_sys::window()@@ -594,6 +656,73 @@ impl Platform for WebPlatform {}}+/// Maps a `navigator.clipboard.read()` rejection to a [`ClipboardReadError`].+///+/// Only `NotAllowedError` and `SecurityError` mean the user or browser+/// refused access; other rejections (e.g. `DataError`) indicate content the+/// clipboard could not represent, where "allow clipboard access" guidance+/// would mislead.+fn clipboard_read_rejection_error(error: JsValue) -> ClipboardReadError {+ match js_error_name(&error).as_deref() {+ Some("NotAllowedError") | Some("SecurityError") => {+ ClipboardReadError::Denied(js_error_message(&error))+ }+ _ => ClipboardReadError::UnsupportedContent,+ }+}++async fn read_clipboard_item_blob(+ item: &web_sys::ClipboardItem,+ mime_type: &str,+) -> Result<web_sys::Blob, JsValue> {+ let blob = wasm_bindgen_futures::JsFuture::from(item.get_type(mime_type)).await?;+ Ok(blob.unchecked_into())+}++async fn read_clipboard_item_text(+ item: &web_sys::ClipboardItem,+ mime_type: &str,+) -> Result<String, JsValue> {+ let blob = read_clipboard_item_blob(item, mime_type).await?;+ let text = wasm_bindgen_futures::JsFuture::from(blob.text()).await?;+ text.as_string()+ .ok_or_else(|| JsValue::from_str("blob text is not a string"))+}++async fn read_clipboard_item_bytes(+ item: &web_sys::ClipboardItem,+ mime_type: &str,+) -> Result<Vec<u8>, JsValue> {+ let blob = read_clipboard_item_blob(item, mime_type).await?;+ read_blob_bytes(&blob).await+}++pub(crate) async fn read_blob_bytes(blob: &web_sys::Blob) -> Result<Vec<u8>, JsValue> {+ let buffer = wasm_bindgen_futures::JsFuture::from(blob.array_buffer()).await?;+ Ok(js_sys::Uint8Array::new(&buffer).to_vec())+}++fn js_error_name(error: &JsValue) -> Option<String> {+ js_sys::Reflect::get(error, &"name".into())+ .ok()+ .and_then(|name| name.as_string())+}++pub(crate) fn js_error_message(error: &JsValue) -> String {+ js_sys::Reflect::get(error, &"message".into())+ .ok()+ .and_then(|message| message.as_string())+ .or_else(|| error.as_string())+ .unwrap_or_else(|| "unknown browser error".to_string())+}++fn log_clipboard_entry_error(mime_type: &str, error: &JsValue) {+ log::warn!(+ "failed to read clipboard entry with type {mime_type}: {}",+ js_error_message(error)+ );+}+fn cursor_restore_listeners(browser_window: &web_sys::Window,cursor_visible: Rc<Cell<bool>>,