Skip to content

Virtual whiteboard for sketching hand-drawn like diagrams

feat: Added Copy/Paste from Google Docs (#7136)

Lakshya Satpal63650f82d134
diff --git a/src/clipboard.test.ts b/src/clipboard.test.ts
index 142038e8e8..f1717464b5 100644
--- a/src/clipboard.test.ts
+++ b/src/clipboard.test.ts
@@ -1,26 +1,21 @@
import { parseClipboard } from "./clipboard";
+import { createPasteEvent } from "./tests/test-utils";
describe("Test parseClipboard", () => {
it("should parse valid json correctly", async () => {
let text = "123";
- let clipboardData = await parseClipboard({
- //@ts-ignore
- clipboardData: {
- getData: () => text,
- },
- });
+ let clipboardData = await parseClipboard(
+ createPasteEvent({ "text/plain": text }),
+ );
expect(clipboardData.text).toBe(text);
text = "[123]";
- clipboardData = await parseClipboard({
- //@ts-ignore
- clipboardData: {
- getData: () => text,
- },
- });
+ clipboardData = await parseClipboard(
+ createPasteEvent({ "text/plain": text }),
+ );
expect(clipboardData.text).toBe(text);
});
diff --git a/src/clipboard.ts b/src/clipboard.ts
index c0deb93af2..bc2167a094 100644
--- a/src/clipboard.ts
+++ b/src/clipboard.ts
@@ -18,11 +18,14 @@ type ElementsClipboard = {
files: BinaryFiles | undefined;
};
+export type PastedMixedContent = { type: "text" | "imageUrl"; value: string }[];
+
export interface ClipboardData {
spreadsheet?: Spreadsheet;
elements?: readonly ExcalidrawElement[];
files?: BinaryFiles;
text?: string;
+ mixedContent?: PastedMixedContent;
errorMessage?: string;
programmaticAPI?: boolean;
}
@@ -142,22 +145,74 @@ const parsePotentialSpreadsheet = (
return null;
};
+/** internal, specific to parsing paste events. Do not reuse. */
+function parseHTMLTree(el: ChildNode) {
+ let result: PastedMixedContent = [];
+ for (const node of el.childNodes) {
+ if (node.nodeType === 3) {
+ const text = node.textContent?.trim();
+ if (text) {
+ result.push({ type: "text", value: text });
+ }
+ } else if (node instanceof HTMLImageElement) {
+ const url = node.getAttribute("src");
+ if (url && url.startsWith("http")) {
+ result.push({ type: "imageUrl", value: url });
+ }
+ } else {
+ result = result.concat(parseHTMLTree(node));
+ }
+ }
+ return result;
+}
+
+const maybeParseHTMLPaste = (event: ClipboardEvent) => {
+ const html = event.clipboardData?.getData("text/html");
+
+ if (!html) {
+ return null;
+ }
+
+ try {
+ const doc = new DOMParser().parseFromString(html, "text/html");
+
+ const content = parseHTMLTree(doc.body);
+
+ if (content.length) {
+ return content;
+ }
+ } catch (error: any) {
+ console.error(`error in parseHTMLFromPaste: ${error.message}`);
+ }
+
+ return null;
+};
+
/**
* Retrieves content from system clipboard (either from ClipboardEvent or
* via async clipboard API if supported)
*/
-export const getSystemClipboard = async (
+const getSystemClipboard = async (
event: ClipboardEvent | null,
-): Promise<string> => {
+ isPlainPaste = false,
+): Promise<
+ | { type: "text"; value: string }
+ | { type: "mixedContent"; value: PastedMixedContent }
+> => {
try {
+ const mixedContent = !isPlainPaste && event && maybeParseHTMLPaste(event);
+ if (mixedContent) {
+ return { type: "mixedContent", value: mixedContent };
+ }
+
const text = event
? event.clipboardData?.getData("text/plain")
: probablySupportsClipboardReadText &&
(await navigator.clipboard.readText());
- return (text || "").trim();
+ return { type: "text", value: (text || "").trim() };
} catch {
- return "";
+ return { type: "text", value: "" };
}
};
@@ -168,14 +223,20 @@ export const parseClipboard = async (
event: ClipboardEvent | null,
isPlainPaste = false,
): Promise<ClipboardData> => {
- const systemClipboard = await getSystemClipboard(event);
+ const systemClipboard = await getSystemClipboard(event, isPlainPaste);
+
+ if (systemClipboard.type === "mixedContent") {
+ return {
+ mixedContent: systemClipboard.value,
+ };
+ }
// if system clipboard empty, couldn't be resolved, or contains previously
// copied excalidraw scene as SVG, fall back to previously copied excalidraw
// elements
if (
!systemClipboard ||
- (!isPlainPaste && systemClipboard.includes(SVG_EXPORT_TAG))
+ (!isPlainPaste && systemClipboard.value.includes(SVG_EXPORT_TAG))
) {
return getAppClipboard();
}
@@ -183,7 +244,7 @@ export const parseClipboard = async (
// if system clipboard contains spreadsheet, use it even though it's
// technically possible it's staler than in-app clipboard
const spreadsheetResult =
- !isPlainPaste && parsePotentialSpreadsheet(systemClipboard);
+ !isPlainPaste && parsePotentialSpreadsheet(systemClipboard.value);
if (spreadsheetResult) {
return spreadsheetResult;
@@ -192,7 +253,7 @@ export const parseClipboard = async (
const appClipboardData = getAppClipboard();
try {
- const systemClipboardData = JSON.parse(systemClipboard);
+ const systemClipboardData = JSON.parse(systemClipboard.value);
const programmaticAPI =
systemClipboardData.type === EXPORT_DATA_TYPES.excalidrawClipboardWithAPI;
if (clipboardContainsElements(systemClipboardData)) {
@@ -216,7 +277,7 @@ export const parseClipboard = async (
? JSON.stringify(appClipboardData.elements, null, 2)
: undefined,
}
- : { text: systemClipboard };
+ : { text: systemClipboard.value };
};
export const copyBlobToClipboardAsPng = async (blob: Blob | Promise<Blob>) => {
diff --git a/src/components/App.tsx b/src/components/App.tsx
index c9d58b35f1..9aa23d62b3 100644
--- a/src/components/App.tsx
+++ b/src/components/App.tsx
@@ -47,7 +47,7 @@ import {
isEraserActive,
isHandToolActive,
} from "../appState";
-import { parseClipboard } from "../clipboard";
+import { PastedMixedContent, parseClipboard } from "../clipboard";
import {
APP_NAME,
CURSOR_TYPE,
@@ -275,6 +275,7 @@ import {
generateIdFromFile,
getDataURL,
getFileFromEvent,
+ ImageURLToFile,
isImageFileHandle,
isSupportedImageFile,
loadSceneOrLibraryFromBlob,
@@ -2183,29 +2184,37 @@ class App extends React.Component<AppProps, AppState> {
return;
}
+ const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(
+ {
+ clientX: this.lastViewportPosition.x,
+ clientY: this.lastViewportPosition.y,
+ },
+ this.state,
+ );
+
// must be called in the same frame (thus before any awaits) as the paste
// event else some browsers (FF...) will clear the clipboardData
// (something something security)
let file = event?.clipboardData?.files[0];
const data = await parseClipboard(event, isPlainPaste);
- if (!file && data.text && !isPlainPaste) {
- const string = data.text.trim();
- if (string.startsWith("<svg") && string.endsWith("</svg>")) {
- // ignore SVG validation/normalization which will be done during image
- // initialization
- file = SVGStringToFile(string);
+ if (!file && !isPlainPaste) {
+ if (data.mixedContent) {
+ return this.addElementsFromMixedContentPaste(data.mixedContent, {
+ isPlainPaste,
+ sceneX,
+ sceneY,
+ });
+ } else if (data.text) {
+ const string = data.text.trim();
+ if (string.startsWith("<svg") && string.endsWith("</svg>")) {
+ // ignore SVG validation/normalization which will be done during image
+ // initialization
+ file = SVGStringToFile(string);
+ }
}
}
- const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(
- {
- clientX: this.lastViewportPosition.x,
- clientY: this.lastViewportPosition.y,
- },
- this.state,
- );
-
// prefer spreadsheet data over image file (MS Office/Libre Office)
if (isSupportedImageFile(file) && !data.spreadsheet) {
const imageElement = this.createImageElement({ sceneX, sceneY });
@@ -2259,6 +2268,7 @@ class App extends React.Component<AppProps, AppState> {
});
} else if (data.text) {
const maybeUrl = extractSrc(data.text);
+
if (
!isPlainPaste &&
embeddableURLValidator(maybeUrl, this.props.validateEmbeddable) &&
@@ -2393,6 +2403,85 @@ class App extends React.Component<AppProps, AppState> {
this.setActiveTool({ type: "selection" });
};
+ // TODO rewrite this to paste both text & images at the same time if
+ // pasted data contains both
+ private async addElementsFromMixedContentPaste(
+ mixedContent: PastedMixedContent,
+ {
+ isPlainPaste,
+ sceneX,
+ sceneY,
+ }: { isPlainPaste: boolean; sceneX: number; sceneY: number },
+ ) {
+ if (
+ !isPlainPaste &&
+ mixedContent.some((node) => node.type === "imageUrl")
+ ) {
+ const imageURLs = mixedContent
+ .filter((node) => node.type === "imageUrl")
+ .map((node) => node.value);
+ const responses = await Promise.all(
+ imageURLs.map(async (url) => {
+ try {
+ return { file: await ImageURLToFile(url) };
+ } catch (error: any) {
+ return { errorMessage: error.message as string };
+ }
+ }),
+ );
+ let y = sceneY;
+ let firstImageYOffsetDone = false;
+ const nextSelectedIds: Record<ExcalidrawElement["id"], true> = {};
+ for (const response of responses) {
+ if (response.file) {
+ const imageElement = this.createImageElement({
+ sceneX,
+ sceneY: y,
+ });
+
+ const initializedImageElement = await this.insertImageElement(
+ imageElement,
+ response.file,
+ );
+ if (initializedImageElement) {
+ // vertically center first image in the batch
+ if (!firstImageYOffsetDone) {
+ firstImageYOffsetDone = true;
+ y -= initializedImageElement.height / 2;
+ }
+ // hack to reset the `y` coord because we vertically center during
+ // insertImageElement
+ mutateElement(initializedImageElement, { y }, false);
+
+ y = imageElement.y + imageElement.height + 25;
+
+ nextSelectedIds[imageElement.id] = true;
+ }
+ }
+ }
+
+ this.setState({
+ selectedElementIds: makeNextSelectedElementIds(
+ nextSelectedIds,
+ this.state,
+ ),
+ });
+
+ const error = responses.find((response) => !!response.errorMessage);
+ if (error && error.errorMessage) {
+ this.setState({ errorMessage: error.errorMessage });
+ }
+ } else {
+ const textNodes = mixedContent.filter((node) => node.type === "text");
+ if (textNodes.length) {
+ this.addTextFromPaste(
+ textNodes.map((node) => node.value).join("\n\n"),
+ isPlainPaste,
+ );
+ }
+ }
+ }
+
private addTextFromPaste(text: string, isPlainPaste = false) {
const { x, y } = viewportCoordsToSceneCoords(
{
@@ -4401,6 +4490,7 @@ class App extends React.Component<AppProps, AppState> {
setCursor(this.interactiveCanvas, CURSOR_TYPE.AUTO);
}
}
+
private handleCanvasPointerDown = (
event: React.PointerEvent<HTMLElement>,
) => {
@@ -7302,7 +7392,7 @@ class App extends React.Component<AppProps, AppState> {
this.scene.addNewElement(imageElement);
try {
- await this.initializeImage({
+ return await this.initializeImage({
imageFile,
imageElement,
showCursorImagePreview,
@@ -7315,6 +7405,7 @@ class App extends React.Component<AppProps, AppState> {
this.setState({
errorMessage: error.message || t("errors.imageInsertError"),
});
+ return null;
}
};
diff --git a/src/data/blob.ts b/src/data/blob.ts
index cb2c051972..81ce340fe1 100644
--- a/src/data/blob.ts
+++ b/src/data/blob.ts
@@ -327,6 +327,31 @@ export const SVGStringToFile = (SVGString: string, filename: string = "") => {
}) as File & { type: typeof MIME_TYPES.svg };
};
+export const ImageURLToFile = async (
+ imageUrl: string,
+ filename: string = "",
+): Promise<File | undefined> => {
+ let response;
+ try {
+ response = await fetch(imageUrl);
+ } catch (error: any) {
+ throw new Error(t("errors.failedToFetchImage"));
+ }
+
+ if (!response.ok) {
+ throw new Error(t("errors.failedToFetchImage"));
+ }
+
+ const blob = await response.blob();
+
+ if (blob.type && isSupportedImageFile(blob)) {
+ const name = filename || blob.name || "";
+ return new File([blob], name, { type: blob.type });
+ }
+
+ throw new Error(t("errors.unsupportedFileType"));
+};
+
export const getFileFromEvent = async (
event: React.DragEvent<HTMLDivElement>,
) => {
diff --git a/src/element/embeddable.ts b/src/element/embeddable.ts
index 4aa6f0fdc0..615fb99a3e 100644
--- a/src/element/embeddable.ts
+++ b/src/element/embeddable.ts
@@ -28,6 +28,7 @@ const embeddedLinkCache = new Map<string, EmbeddedLink>();
const RE_YOUTUBE =
/^(?:http(?:s)?:\/\/)?(?:www\.)?youtu(?:be\.com|\.be)\/(embed\/|watch\?v=|shorts\/|playlist\?list=|embed\/videoseries\?list=)?([a-zA-Z0-9_-]+)(?:\?t=|&t=|\?start=|&start=)?([a-zA-Z0-9_-]+)?[^\s]*$/;
+
const RE_VIMEO =
/^(?:http(?:s)?:\/\/)?(?:(?:w){3}.)?(?:player\.)?vimeo\.com\/(?:video\/)?([^?\s]+)(?:\?.*)?$/;
const RE_FIGMA = /^https:\/\/(?:www\.)?figma\.com/;
diff --git a/src/locales/en.json b/src/locales/en.json
index f2e6b601a0..19dbf81f03 100644
--- a/src/locales/en.json
+++ b/src/locales/en.json
@@ -203,6 +203,7 @@
"imageInsertError": "Couldn't insert image. Try again later...",
"fileTooBig": "File is too big. Maximum allowed size is {{maxSize}}.",
"svgImageInsertError": "Couldn't insert SVG image. The SVG markup looks invalid.",
+ "failedToFetchImage": "Failed to fetch image.",
"invalidSVGString": "Invalid SVG.",
"cannotResolveCollabServer": "Couldn't connect to the collab server. Please reload the page and try again.",
"importLibraryError": "Couldn't load library",
diff --git a/src/tests/clipboard.test.tsx b/src/tests/clipboard.test.tsx
index 4a55e0689f..0ea480ad64 100644
--- a/src/tests/clipboard.test.tsx
+++ b/src/tests/clipboard.test.tsx
@@ -35,22 +35,14 @@ vi.mock("../keys.ts", async (importOriginal) => {
};
});
-const setClipboardText = (text: string) => {
- Object.assign(navigator, {
- clipboard: {
- readText: () => text,
- },
+const sendPasteEvent = (text: string) => {
+ const clipboardEvent = createPasteEvent({
+ "text/plain": text,
});
-};
-
-const sendPasteEvent = (text?: string) => {
- const clipboardEvent = createPasteEvent(
- text || (() => window.navigator.clipboard.readText()),
- );
document.dispatchEvent(clipboardEvent);
};
-const pasteWithCtrlCmdShiftV = (text?: string) => {
+const pasteWithCtrlCmdShiftV = (text: string) => {
Keyboard.withModifierKeys({ ctrl: true, shift: true }, () => {
//triggering keydown with an empty clipboard
Keyboard.keyPress(KEYS.V);
@@ -59,7 +51,7 @@ const pasteWithCtrlCmdShiftV = (text?: string) => {
});
};
-const pasteWithCtrlCmdV = (text?: string) => {
+const pasteWithCtrlCmdV = (text: string) => {
Keyboard.withModifierKeys({ ctrl: true }, () => {
//triggering keydown with an empty clipboard
Keyboard.keyPress(KEYS.V);
@@ -86,7 +78,6 @@ beforeEach(async () => {
initialData={{ appState: { zoom: { value: 1 as NormalizedZoomValue } } }}
/>,
);
- setClipboardText("");
Object.assign(document, {
elementFromPoint: () => GlobalTestState.canvas,
});
@@ -120,8 +111,7 @@ describe("general paste behavior", () => {
describe("paste text as single lines", () => {
it("should create an element for each line when copying with Ctrl/Cmd+V", async () => {
const text = "sajgfakfn\naaksfnknas\nakefnkasf";
- setClipboardText(text);
- pasteWithCtrlCmdV();
+ pasteWithCtrlCmdV(text);
await waitFor(() => {
expect(h.elements.length).toEqual(text.split("\n").length);
});
@@ -129,8 +119,7 @@ describe("paste text as single lines", () => {
it("should ignore empty lines when creating an element for each line", async () => {
const text = "\n\nsajgfakfn\n\n\naaksfnknas\n\nakefnkasf\n\n\n";
- setClipboardText(text);
- pasteWithCtrlCmdV();
+ pasteWithCtrlCmdV(text);
await waitFor(() => {
expect(h.elements.length).toEqual(3);
});
@@ -138,8 +127,7 @@ describe("paste text as single lines", () => {
it("should not create any element if clipboard has only new lines", async () => {
const text = "\n\n\n\n\n";
- setClipboardText(text);
- pasteWithCtrlCmdV();
+ pasteWithCtrlCmdV(text);
await waitFor(async () => {
await sleep(50); // elements lenght will always be zero if we don't wait, since paste is async
expect(h.elements.length).toEqual(0);
@@ -155,8 +143,7 @@ describe("paste text as single lines", () => {
) +
10 / h.app.state.zoom.value;
mouse.moveTo(100, 100);
- setClipboardText(text);
- pasteWithCtrlCmdV();
+ pasteWithCtrlCmdV(text);
await waitFor(async () => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [fx, firstElY] = getElementBounds(h.elements[0]);
@@ -177,8 +164,7 @@ describe("paste text as single lines", () => {
) +
10 / h.app.state.zoom.value;
mouse.moveTo(100, 100);
- setClipboardText(text);
- pasteWithCtrlCmdV();
+ pasteWithCtrlCmdV(text);
await waitFor(async () => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [fx, firstElY] = getElementBounds(h.elements[0]);
@@ -192,16 +178,14 @@ describe("paste text as single lines", () => {
describe("paste text as a single element", () => {
it("should create single text element when copying text with Ctrl/Cmd+Shift+V", async () => {
const text = "sajgfakfn\naaksfnknas\nakefnkasf";
- setClipboardText(text);
- pasteWithCtrlCmdShiftV();
+ pasteWithCtrlCmdShiftV(text);
await waitFor(() => {
expect(h.elements.length).toEqual(1);
});
});
it("should not create any element when only new lines in clipboard", async () => {
const text = "\n\n\n\n";
- setClipboardText(text);
- pasteWithCtrlCmdShiftV();
+ pasteWithCtrlCmdShiftV(text);
await waitFor(async () => {
await sleep(50);
expect(h.elements.length).toEqual(0);
@@ -243,8 +227,7 @@ describe("Paste bound text container", () => {
type: "excalidraw/clipboard",
elements: [container, textElement],
});
- setClipboardText(data);
- pasteWithCtrlCmdShiftV();
+ pasteWithCtrlCmdShiftV(data);
await waitFor(async () => {
await sleep(1);
@@ -266,8 +249,7 @@ describe("Paste bound text container", () => {
textElement,
],
});
- setClipboardText(data);
- pasteWithCtrlCmdShiftV();
+ pasteWithCtrlCmdShiftV(data);
await waitFor(async () => {
await sleep(1);
diff --git a/src/tests/flip.test.tsx b/src/tests/flip.test.tsx
index b80b25f2d4..982105e979 100644
--- a/src/tests/flip.test.tsx
+++ b/src/tests/flip.test.tsx
@@ -727,7 +727,7 @@ describe("freedraw", () => {
describe("image", () => {
const createImage = async () => {
const sendPasteEvent = (file?: File) => {
- const clipboardEvent = createPasteEvent("", file ? [file] : []);
+ const clipboardEvent = createPasteEvent({}, file ? [file] : []);
document.dispatchEvent(clipboardEvent);
};
diff --git a/src/tests/test-utils.ts b/src/tests/test-utils.ts
index fed6b4897d..0f80182a94 100644
--- a/src/tests/test-utils.ts
+++ b/src/tests/test-utils.ts
@@ -208,10 +208,8 @@ export const assertSelectedElements = (
expect(selectedElementIds).toEqual(expect.arrayContaining(ids));
};
-export const createPasteEvent = (
- text:
- | string
- | /* getData function */ ((type: string) => string | Promise<string>),
+export const createPasteEvent = <T extends "text/plain" | "text/html">(
+ items: Record<T, string>,
files?: File[],
) => {
return Object.assign(
@@ -222,11 +220,12 @@ export const createPasteEvent = (
}),
{
clipboardData: {
- getData: typeof text === "string" ? () => text : text,
+ getData: (type: string) =>
+ (items as Record<string, string>)[type] || "",
files: files || [],
},
},
- );
+ ) as any as ClipboardEvent;
};
export const toggleMenu = (container: HTMLElement) => {