Virtual whiteboard for sketching hand-drawn like diagrams
feat: Added Copy/Paste from Google Docs (#7136)
Lakshya Satpal63650f82d134
diff --git a/src/clipboard.test.ts b/src/clipboard.test.tsindex 142038e8e8..f1717464b5 100644--- a/src/clipboard.test.ts+++ b/src/clipboard.test.ts@@ -1,26 +1,21 @@import { parseClipboard } from "./clipboard";+import { createPasteEvent } from "./tests/test-utils";describe("Test parseClipboard", () => {it("should parse valid json correctly", async () => {let text = "123";- let clipboardData = await parseClipboard({- //@ts-ignore- clipboardData: {- getData: () => text,- },- });+ let clipboardData = await parseClipboard(+ createPasteEvent({ "text/plain": text }),+ );expect(clipboardData.text).toBe(text);text = "[123]";- clipboardData = await parseClipboard({- //@ts-ignore- clipboardData: {- getData: () => text,- },- });+ clipboardData = await parseClipboard(+ createPasteEvent({ "text/plain": text }),+ );expect(clipboardData.text).toBe(text);});diff --git a/src/clipboard.ts b/src/clipboard.tsindex c0deb93af2..bc2167a094 100644--- a/src/clipboard.ts+++ b/src/clipboard.ts@@ -18,11 +18,14 @@ type ElementsClipboard = {files: BinaryFiles | undefined;};+export type PastedMixedContent = { type: "text" | "imageUrl"; value: string }[];+export interface ClipboardData {spreadsheet?: Spreadsheet;elements?: readonly ExcalidrawElement[];files?: BinaryFiles;text?: string;+ mixedContent?: PastedMixedContent;errorMessage?: string;programmaticAPI?: boolean;}@@ -142,22 +145,74 @@ const parsePotentialSpreadsheet = (return null;};+/** internal, specific to parsing paste events. Do not reuse. */+function parseHTMLTree(el: ChildNode) {+ let result: PastedMixedContent = [];+ for (const node of el.childNodes) {+ if (node.nodeType === 3) {+ const text = node.textContent?.trim();+ if (text) {+ result.push({ type: "text", value: text });+ }+ } else if (node instanceof HTMLImageElement) {+ const url = node.getAttribute("src");+ if (url && url.startsWith("http")) {+ result.push({ type: "imageUrl", value: url });+ }+ } else {+ result = result.concat(parseHTMLTree(node));+ }+ }+ return result;+}++const maybeParseHTMLPaste = (event: ClipboardEvent) => {+ const html = event.clipboardData?.getData("text/html");++ if (!html) {+ return null;+ }++ try {+ const doc = new DOMParser().parseFromString(html, "text/html");++ const content = parseHTMLTree(doc.body);++ if (content.length) {+ return content;+ }+ } catch (error: any) {+ console.error(`error in parseHTMLFromPaste: ${error.message}`);+ }++ return null;+};+/*** Retrieves content from system clipboard (either from ClipboardEvent or* via async clipboard API if supported)*/-export const getSystemClipboard = async (+const getSystemClipboard = async (event: ClipboardEvent | null,-): Promise<string> => {+ isPlainPaste = false,+): Promise<+ | { type: "text"; value: string }+ | { type: "mixedContent"; value: PastedMixedContent }+> => {try {+ const mixedContent = !isPlainPaste && event && maybeParseHTMLPaste(event);+ if (mixedContent) {+ return { type: "mixedContent", value: mixedContent };+ }+const text = event? event.clipboardData?.getData("text/plain"): probablySupportsClipboardReadText &&(await navigator.clipboard.readText());- return (text || "").trim();+ return { type: "text", value: (text || "").trim() };} catch {- return "";+ return { type: "text", value: "" };}};@@ -168,14 +223,20 @@ export const parseClipboard = async (event: ClipboardEvent | null,isPlainPaste = false,): Promise<ClipboardData> => {- const systemClipboard = await getSystemClipboard(event);+ const systemClipboard = await getSystemClipboard(event, isPlainPaste);++ if (systemClipboard.type === "mixedContent") {+ return {+ mixedContent: systemClipboard.value,+ };+ }// if system clipboard empty, couldn't be resolved, or contains previously// copied excalidraw scene as SVG, fall back to previously copied excalidraw// elementsif (!systemClipboard ||- (!isPlainPaste && systemClipboard.includes(SVG_EXPORT_TAG))+ (!isPlainPaste && systemClipboard.value.includes(SVG_EXPORT_TAG))) {return getAppClipboard();}@@ -183,7 +244,7 @@ export const parseClipboard = async (// if system clipboard contains spreadsheet, use it even though it's// technically possible it's staler than in-app clipboardconst spreadsheetResult =- !isPlainPaste && parsePotentialSpreadsheet(systemClipboard);+ !isPlainPaste && parsePotentialSpreadsheet(systemClipboard.value);if (spreadsheetResult) {return spreadsheetResult;@@ -192,7 +253,7 @@ export const parseClipboard = async (const appClipboardData = getAppClipboard();try {- const systemClipboardData = JSON.parse(systemClipboard);+ const systemClipboardData = JSON.parse(systemClipboard.value);const programmaticAPI =systemClipboardData.type === EXPORT_DATA_TYPES.excalidrawClipboardWithAPI;if (clipboardContainsElements(systemClipboardData)) {@@ -216,7 +277,7 @@ export const parseClipboard = async (? JSON.stringify(appClipboardData.elements, null, 2): undefined,}- : { text: systemClipboard };+ : { text: systemClipboard.value };};export const copyBlobToClipboardAsPng = async (blob: Blob | Promise<Blob>) => {diff --git a/src/components/App.tsx b/src/components/App.tsxindex c9d58b35f1..9aa23d62b3 100644--- a/src/components/App.tsx+++ b/src/components/App.tsx@@ -47,7 +47,7 @@ import {isEraserActive,isHandToolActive,} from "../appState";-import { parseClipboard } from "../clipboard";+import { PastedMixedContent, parseClipboard } from "../clipboard";import {APP_NAME,CURSOR_TYPE,@@ -275,6 +275,7 @@ import {generateIdFromFile,getDataURL,getFileFromEvent,+ ImageURLToFile,isImageFileHandle,isSupportedImageFile,loadSceneOrLibraryFromBlob,@@ -2183,29 +2184,37 @@ class App extends React.Component<AppProps, AppState> {return;}+ const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(+ {+ clientX: this.lastViewportPosition.x,+ clientY: this.lastViewportPosition.y,+ },+ this.state,+ );+// must be called in the same frame (thus before any awaits) as the paste// event else some browsers (FF...) will clear the clipboardData// (something something security)let file = event?.clipboardData?.files[0];const data = await parseClipboard(event, isPlainPaste);- if (!file && data.text && !isPlainPaste) {- const string = data.text.trim();- if (string.startsWith("<svg") && string.endsWith("</svg>")) {- // ignore SVG validation/normalization which will be done during image- // initialization- file = SVGStringToFile(string);+ if (!file && !isPlainPaste) {+ if (data.mixedContent) {+ return this.addElementsFromMixedContentPaste(data.mixedContent, {+ isPlainPaste,+ sceneX,+ sceneY,+ });+ } else if (data.text) {+ const string = data.text.trim();+ if (string.startsWith("<svg") && string.endsWith("</svg>")) {+ // ignore SVG validation/normalization which will be done during image+ // initialization+ file = SVGStringToFile(string);+ }}}- const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(- {- clientX: this.lastViewportPosition.x,- clientY: this.lastViewportPosition.y,- },- this.state,- );-// prefer spreadsheet data over image file (MS Office/Libre Office)if (isSupportedImageFile(file) && !data.spreadsheet) {const imageElement = this.createImageElement({ sceneX, sceneY });@@ -2259,6 +2268,7 @@ class App extends React.Component<AppProps, AppState> {});} else if (data.text) {const maybeUrl = extractSrc(data.text);+if (!isPlainPaste &&embeddableURLValidator(maybeUrl, this.props.validateEmbeddable) &&@@ -2393,6 +2403,85 @@ class App extends React.Component<AppProps, AppState> {this.setActiveTool({ type: "selection" });};+ // TODO rewrite this to paste both text & images at the same time if+ // pasted data contains both+ private async addElementsFromMixedContentPaste(+ mixedContent: PastedMixedContent,+ {+ isPlainPaste,+ sceneX,+ sceneY,+ }: { isPlainPaste: boolean; sceneX: number; sceneY: number },+ ) {+ if (+ !isPlainPaste &&+ mixedContent.some((node) => node.type === "imageUrl")+ ) {+ const imageURLs = mixedContent+ .filter((node) => node.type === "imageUrl")+ .map((node) => node.value);+ const responses = await Promise.all(+ imageURLs.map(async (url) => {+ try {+ return { file: await ImageURLToFile(url) };+ } catch (error: any) {+ return { errorMessage: error.message as string };+ }+ }),+ );+ let y = sceneY;+ let firstImageYOffsetDone = false;+ const nextSelectedIds: Record<ExcalidrawElement["id"], true> = {};+ for (const response of responses) {+ if (response.file) {+ const imageElement = this.createImageElement({+ sceneX,+ sceneY: y,+ });++ const initializedImageElement = await this.insertImageElement(+ imageElement,+ response.file,+ );+ if (initializedImageElement) {+ // vertically center first image in the batch+ if (!firstImageYOffsetDone) {+ firstImageYOffsetDone = true;+ y -= initializedImageElement.height / 2;+ }+ // hack to reset the `y` coord because we vertically center during+ // insertImageElement+ mutateElement(initializedImageElement, { y }, false);++ y = imageElement.y + imageElement.height + 25;++ nextSelectedIds[imageElement.id] = true;+ }+ }+ }++ this.setState({+ selectedElementIds: makeNextSelectedElementIds(+ nextSelectedIds,+ this.state,+ ),+ });++ const error = responses.find((response) => !!response.errorMessage);+ if (error && error.errorMessage) {+ this.setState({ errorMessage: error.errorMessage });+ }+ } else {+ const textNodes = mixedContent.filter((node) => node.type === "text");+ if (textNodes.length) {+ this.addTextFromPaste(+ textNodes.map((node) => node.value).join("\n\n"),+ isPlainPaste,+ );+ }+ }+ }+private addTextFromPaste(text: string, isPlainPaste = false) {const { x, y } = viewportCoordsToSceneCoords({@@ -4401,6 +4490,7 @@ class App extends React.Component<AppProps, AppState> {setCursor(this.interactiveCanvas, CURSOR_TYPE.AUTO);}}+private handleCanvasPointerDown = (event: React.PointerEvent<HTMLElement>,) => {@@ -7302,7 +7392,7 @@ class App extends React.Component<AppProps, AppState> {this.scene.addNewElement(imageElement);try {- await this.initializeImage({+ return await this.initializeImage({imageFile,imageElement,showCursorImagePreview,@@ -7315,6 +7405,7 @@ class App extends React.Component<AppProps, AppState> {this.setState({errorMessage: error.message || t("errors.imageInsertError"),});+ return null;}};diff --git a/src/data/blob.ts b/src/data/blob.tsindex cb2c051972..81ce340fe1 100644--- a/src/data/blob.ts+++ b/src/data/blob.ts@@ -327,6 +327,31 @@ export const SVGStringToFile = (SVGString: string, filename: string = "") => {}) as File & { type: typeof MIME_TYPES.svg };};+export const ImageURLToFile = async (+ imageUrl: string,+ filename: string = "",+): Promise<File | undefined> => {+ let response;+ try {+ response = await fetch(imageUrl);+ } catch (error: any) {+ throw new Error(t("errors.failedToFetchImage"));+ }++ if (!response.ok) {+ throw new Error(t("errors.failedToFetchImage"));+ }++ const blob = await response.blob();++ if (blob.type && isSupportedImageFile(blob)) {+ const name = filename || blob.name || "";+ return new File([blob], name, { type: blob.type });+ }++ throw new Error(t("errors.unsupportedFileType"));+};+export const getFileFromEvent = async (event: React.DragEvent<HTMLDivElement>,) => {diff --git a/src/element/embeddable.ts b/src/element/embeddable.tsindex 4aa6f0fdc0..615fb99a3e 100644--- a/src/element/embeddable.ts+++ b/src/element/embeddable.ts@@ -28,6 +28,7 @@ const embeddedLinkCache = new Map<string, EmbeddedLink>();const RE_YOUTUBE =/^(?:http(?:s)?:\/\/)?(?:www\.)?youtu(?:be\.com|\.be)\/(embed\/|watch\?v=|shorts\/|playlist\?list=|embed\/videoseries\?list=)?([a-zA-Z0-9_-]+)(?:\?t=|&t=|\?start=|&start=)?([a-zA-Z0-9_-]+)?[^\s]*$/;+const RE_VIMEO =/^(?:http(?:s)?:\/\/)?(?:(?:w){3}.)?(?:player\.)?vimeo\.com\/(?:video\/)?([^?\s]+)(?:\?.*)?$/;const RE_FIGMA = /^https:\/\/(?:www\.)?figma\.com/;diff --git a/src/locales/en.json b/src/locales/en.jsonindex f2e6b601a0..19dbf81f03 100644--- a/src/locales/en.json+++ b/src/locales/en.json@@ -203,6 +203,7 @@"imageInsertError": "Couldn't insert image. Try again later...","fileTooBig": "File is too big. Maximum allowed size is {{maxSize}}.","svgImageInsertError": "Couldn't insert SVG image. The SVG markup looks invalid.",+ "failedToFetchImage": "Failed to fetch image.","invalidSVGString": "Invalid SVG.","cannotResolveCollabServer": "Couldn't connect to the collab server. Please reload the page and try again.","importLibraryError": "Couldn't load library",diff --git a/src/tests/clipboard.test.tsx b/src/tests/clipboard.test.tsxindex 4a55e0689f..0ea480ad64 100644--- a/src/tests/clipboard.test.tsx+++ b/src/tests/clipboard.test.tsx@@ -35,22 +35,14 @@ vi.mock("../keys.ts", async (importOriginal) => {};});-const setClipboardText = (text: string) => {- Object.assign(navigator, {- clipboard: {- readText: () => text,- },+const sendPasteEvent = (text: string) => {+ const clipboardEvent = createPasteEvent({+ "text/plain": text,});-};--const sendPasteEvent = (text?: string) => {- const clipboardEvent = createPasteEvent(- text || (() => window.navigator.clipboard.readText()),- );document.dispatchEvent(clipboardEvent);};-const pasteWithCtrlCmdShiftV = (text?: string) => {+const pasteWithCtrlCmdShiftV = (text: string) => {Keyboard.withModifierKeys({ ctrl: true, shift: true }, () => {//triggering keydown with an empty clipboardKeyboard.keyPress(KEYS.V);@@ -59,7 +51,7 @@ const pasteWithCtrlCmdShiftV = (text?: string) => {});};-const pasteWithCtrlCmdV = (text?: string) => {+const pasteWithCtrlCmdV = (text: string) => {Keyboard.withModifierKeys({ ctrl: true }, () => {//triggering keydown with an empty clipboardKeyboard.keyPress(KEYS.V);@@ -86,7 +78,6 @@ beforeEach(async () => {initialData={{ appState: { zoom: { value: 1 as NormalizedZoomValue } } }}/>,);- setClipboardText("");Object.assign(document, {elementFromPoint: () => GlobalTestState.canvas,});@@ -120,8 +111,7 @@ describe("general paste behavior", () => {describe("paste text as single lines", () => {it("should create an element for each line when copying with Ctrl/Cmd+V", async () => {const text = "sajgfakfn\naaksfnknas\nakefnkasf";- setClipboardText(text);- pasteWithCtrlCmdV();+ pasteWithCtrlCmdV(text);await waitFor(() => {expect(h.elements.length).toEqual(text.split("\n").length);});@@ -129,8 +119,7 @@ describe("paste text as single lines", () => {it("should ignore empty lines when creating an element for each line", async () => {const text = "\n\nsajgfakfn\n\n\naaksfnknas\n\nakefnkasf\n\n\n";- setClipboardText(text);- pasteWithCtrlCmdV();+ pasteWithCtrlCmdV(text);await waitFor(() => {expect(h.elements.length).toEqual(3);});@@ -138,8 +127,7 @@ describe("paste text as single lines", () => {it("should not create any element if clipboard has only new lines", async () => {const text = "\n\n\n\n\n";- setClipboardText(text);- pasteWithCtrlCmdV();+ pasteWithCtrlCmdV(text);await waitFor(async () => {await sleep(50); // elements lenght will always be zero if we don't wait, since paste is asyncexpect(h.elements.length).toEqual(0);@@ -155,8 +143,7 @@ describe("paste text as single lines", () => {) +10 / h.app.state.zoom.value;mouse.moveTo(100, 100);- setClipboardText(text);- pasteWithCtrlCmdV();+ pasteWithCtrlCmdV(text);await waitFor(async () => {// eslint-disable-next-line @typescript-eslint/no-unused-varsconst [fx, firstElY] = getElementBounds(h.elements[0]);@@ -177,8 +164,7 @@ describe("paste text as single lines", () => {) +10 / h.app.state.zoom.value;mouse.moveTo(100, 100);- setClipboardText(text);- pasteWithCtrlCmdV();+ pasteWithCtrlCmdV(text);await waitFor(async () => {// eslint-disable-next-line @typescript-eslint/no-unused-varsconst [fx, firstElY] = getElementBounds(h.elements[0]);@@ -192,16 +178,14 @@ describe("paste text as single lines", () => {describe("paste text as a single element", () => {it("should create single text element when copying text with Ctrl/Cmd+Shift+V", async () => {const text = "sajgfakfn\naaksfnknas\nakefnkasf";- setClipboardText(text);- pasteWithCtrlCmdShiftV();+ pasteWithCtrlCmdShiftV(text);await waitFor(() => {expect(h.elements.length).toEqual(1);});});it("should not create any element when only new lines in clipboard", async () => {const text = "\n\n\n\n";- setClipboardText(text);- pasteWithCtrlCmdShiftV();+ pasteWithCtrlCmdShiftV(text);await waitFor(async () => {await sleep(50);expect(h.elements.length).toEqual(0);@@ -243,8 +227,7 @@ describe("Paste bound text container", () => {type: "excalidraw/clipboard",elements: [container, textElement],});- setClipboardText(data);- pasteWithCtrlCmdShiftV();+ pasteWithCtrlCmdShiftV(data);await waitFor(async () => {await sleep(1);@@ -266,8 +249,7 @@ describe("Paste bound text container", () => {textElement,],});- setClipboardText(data);- pasteWithCtrlCmdShiftV();+ pasteWithCtrlCmdShiftV(data);await waitFor(async () => {await sleep(1);diff --git a/src/tests/flip.test.tsx b/src/tests/flip.test.tsxindex b80b25f2d4..982105e979 100644--- a/src/tests/flip.test.tsx+++ b/src/tests/flip.test.tsx@@ -727,7 +727,7 @@ describe("freedraw", () => {describe("image", () => {const createImage = async () => {const sendPasteEvent = (file?: File) => {- const clipboardEvent = createPasteEvent("", file ? [file] : []);+ const clipboardEvent = createPasteEvent({}, file ? [file] : []);document.dispatchEvent(clipboardEvent);};diff --git a/src/tests/test-utils.ts b/src/tests/test-utils.tsindex fed6b4897d..0f80182a94 100644--- a/src/tests/test-utils.ts+++ b/src/tests/test-utils.ts@@ -208,10 +208,8 @@ export const assertSelectedElements = (expect(selectedElementIds).toEqual(expect.arrayContaining(ids));};-export const createPasteEvent = (- text:- | string- | /* getData function */ ((type: string) => string | Promise<string>),+export const createPasteEvent = <T extends "text/plain" | "text/html">(+ items: Record<T, string>,files?: File[],) => {return Object.assign(@@ -222,11 +220,12 @@ export const createPasteEvent = (}),{clipboardData: {- getData: typeof text === "string" ? () => text : text,+ getData: (type: string) =>+ (items as Record<string, string>)[type] || "",files: files || [],},},- );+ ) as any as ClipboardEvent;};export const toggleMenu = (container: HTMLElement) => {