Virtual whiteboard for sketching hand-drawn like diagrams
feat: rewrite d2c to not require token (#8269)
David Luzarb5d7f5b4ba85
diff --git a/excalidraw-app/App.tsx b/excalidraw-app/App.tsxindex a11ea59b32..69d6211293 100644--- a/excalidraw-app/App.tsx+++ b/excalidraw-app/App.tsx@@ -22,7 +22,6 @@ import { t } from "../packages/excalidraw/i18n";import {Excalidraw,LiveCollaborationTrigger,- TTDDialog,TTDDialogTrigger,StoreAction,reconcileElements,@@ -121,6 +120,7 @@ import {import { appThemeAtom, useHandleAppTheme } from "./useHandleAppTheme";import { getPreferredLanguage } from "./app-language/language-detector";import { useAppLangCode } from "./app-language/language-state";+import { AIComponents } from "./components/AI";polyfill();@@ -846,63 +846,8 @@ const ExcalidrawWrapper = () => {)}</OverwriteConfirmDialog><AppFooter />- <TTDDialog- onTextSubmit={async (input) => {- try {- const response = await fetch(- `${- import.meta.env.VITE_APP_AI_BACKEND- }/v1/ai/text-to-diagram/generate`,- {- method: "POST",- headers: {- Accept: "application/json",- "Content-Type": "application/json",- },- body: JSON.stringify({ prompt: input }),- },- );-- const rateLimit = response.headers.has("X-Ratelimit-Limit")- ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)- : undefined;-- const rateLimitRemaining = response.headers.has(- "X-Ratelimit-Remaining",- )- ? parseInt(- response.headers.get("X-Ratelimit-Remaining") || "0",- 10,- )- : undefined;-- const json = await response.json();-- if (!response.ok) {- if (response.status === 429) {- return {- rateLimit,- rateLimitRemaining,- error: new Error(- "Too many requests today, please try again tomorrow!",- ),- };- }+ {excalidrawAPI && <AIComponents excalidrawAPI={excalidrawAPI} />}- throw new Error(json.message || "Generation failed...");- }-- const generatedResponse = json.generatedResponse;- if (!generatedResponse) {- throw new Error("Generation failed...");- }-- return { generatedResponse, rateLimit, rateLimitRemaining };- } catch (err: any) {- throw new Error("Request failed");- }- }}- /><TTDDialogTrigger />{isCollaborating && isOffline && (<div className="collab-offline-warning">diff --git a/excalidraw-app/components/AI.tsx b/excalidraw-app/components/AI.tsxnew file mode 100644index 0000000000..621d6befa7--- /dev/null+++ b/excalidraw-app/components/AI.tsx@@ -0,0 +1,159 @@+import type { ExcalidrawImperativeAPI } from "../../packages/excalidraw/types";+import {+ DiagramToCodePlugin,+ exportToBlob,+ getTextFromElements,+ MIME_TYPES,+ TTDDialog,+} from "../../packages/excalidraw";+import { getDataURL } from "../../packages/excalidraw/data/blob";+import { safelyParseJSON } from "../../packages/excalidraw/utils";++export const AIComponents = ({+ excalidrawAPI,+}: {+ excalidrawAPI: ExcalidrawImperativeAPI;+}) => {+ return (+ <>+ <DiagramToCodePlugin+ generate={async ({ frame, children }) => {+ const appState = excalidrawAPI.getAppState();++ const blob = await exportToBlob({+ elements: children,+ appState: {+ ...appState,+ exportBackground: true,+ viewBackgroundColor: appState.viewBackgroundColor,+ },+ exportingFrame: frame,+ files: excalidrawAPI.getFiles(),+ mimeType: MIME_TYPES.jpg,+ });++ const dataURL = await getDataURL(blob);++ const textFromFrameChildren = getTextFromElements(children);++ const response = await fetch(+ `${+ import.meta.env.VITE_APP_AI_BACKEND+ }/v1/ai/diagram-to-code/generate`,+ {+ method: "POST",+ headers: {+ Accept: "application/json",+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ texts: textFromFrameChildren,+ image: dataURL,+ theme: appState.theme,+ }),+ },+ );++ if (!response.ok) {+ const text = await response.text();+ const errorJSON = safelyParseJSON(text);++ if (!errorJSON) {+ throw new Error(text);+ }++ if (errorJSON.statusCode === 429) {+ return {+ html: `<html>+ <body style="margin: 0; text-align: center">+ <div style="display: flex; align-items: center; justify-content: center; flex-direction: column; height: 100vh; padding: 0 60px">+ <div style="color:red">Too many requests today,</br>please try again tomorrow!</div>+ </br>+ </br>+ <div>You can also try <a href="${+ import.meta.env.VITE_APP_PLUS_LP+ }/plus?utm_source=excalidraw&utm_medium=app&utm_content=d2c" target="_blank" rel="noreferrer noopener">Excalidraw+</a> to get more requests.</div>+ </div>+ </body>+ </html>`,+ };+ }++ throw new Error(errorJSON.message || text);+ }++ try {+ const { html } = await response.json();++ if (!html) {+ throw new Error("Generation failed (invalid response)");+ }+ return {+ html,+ };+ } catch (error: any) {+ throw new Error("Generation failed (invalid response)");+ }+ }}+ />++ <TTDDialog+ onTextSubmit={async (input) => {+ try {+ const response = await fetch(+ `${+ import.meta.env.VITE_APP_AI_BACKEND+ }/v1/ai/text-to-diagram/generate`,+ {+ method: "POST",+ headers: {+ Accept: "application/json",+ "Content-Type": "application/json",+ },+ body: JSON.stringify({ prompt: input }),+ },+ );++ const rateLimit = response.headers.has("X-Ratelimit-Limit")+ ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)+ : undefined;++ const rateLimitRemaining = response.headers.has(+ "X-Ratelimit-Remaining",+ )+ ? parseInt(+ response.headers.get("X-Ratelimit-Remaining") || "0",+ 10,+ )+ : undefined;++ const json = await response.json();++ if (!response.ok) {+ if (response.status === 429) {+ return {+ rateLimit,+ rateLimitRemaining,+ error: new Error(+ "Too many requests today, please try again tomorrow!",+ ),+ };+ }++ throw new Error(json.message || "Generation failed...");+ }++ const generatedResponse = json.generatedResponse;+ if (!generatedResponse) {+ throw new Error("Generation failed...");+ }++ return { generatedResponse, rateLimit, rateLimitRemaining };+ } catch (err: any) {+ throw new Error("Request failed");+ }+ }}+ />+ </>+ );+};diff --git a/packages/excalidraw/actions/actionClipboard.tsx b/packages/excalidraw/actions/actionClipboard.tsxindex e4f998d016..4d34bf968c 100644--- a/packages/excalidraw/actions/actionClipboard.tsx+++ b/packages/excalidraw/actions/actionClipboard.tsx@@ -10,7 +10,7 @@ import {} from "../clipboard";import { actionDeleteSelected } from "./actionDeleteSelected";import { exportCanvas, prepareElementsForExport } from "../data/index";-import { isTextElement } from "../element";+import { getTextFromElements, isTextElement } from "../element";import { t } from "../i18n";import { isFirefox } from "../constants";import { DuplicateIcon, cutIcon, pngIcon, svgIcon } from "../components/icons";@@ -239,16 +239,8 @@ export const copyText = register({includeBoundTextElement: true,});- const text = selectedElements- .reduce((acc: string[], element) => {- if (isTextElement(element)) {- acc.push(element.text);- }- return acc;- }, [])- .join("\n\n");try {- copyTextToSystemClipboard(text);+ copyTextToSystemClipboard(getTextFromElements(selectedElements));} catch (e) {throw new Error(t("errors.copyToSystemClipboardFailed"));}diff --git a/packages/excalidraw/components/Actions.tsx b/packages/excalidraw/components/Actions.tsxindex 91102aef2c..15bddefdd5 100644--- a/packages/excalidraw/components/Actions.tsx+++ b/packages/excalidraw/components/Actions.tsx@@ -45,7 +45,6 @@ import {frameToolIcon,mermaidLogoIcon,laserPointerToolIcon,- OpenAIIcon,MagicIcon,} from "./icons";import { KEYS } from "../keys";@@ -400,7 +399,7 @@ export const ShapesSwitcher = ({>{t("toolBar.mermaidToExcalidraw")}</DropdownMenu.Item>- {app.props.aiEnabled !== false && (+ {app.props.aiEnabled !== false && app.plugins.diagramToCode && (<><DropdownMenu.ItemonSelect={() => app.onMagicframeToolSelect()}@@ -410,20 +409,6 @@ export const ShapesSwitcher = ({{t("toolBar.magicframe")}<DropdownMenu.Item.Badge>AI</DropdownMenu.Item.Badge></DropdownMenu.Item>- <DropdownMenu.Item- onSelect={() => {- trackEvent("ai", "open-settings", "d2c");- app.setOpenDialog({- name: "settings",- source: "settings",- tab: "diagram-to-code",- });- }}- icon={OpenAIIcon}- data-testid="toolbar-magicSettings"- >- {t("toolBar.magicSettings")}- </DropdownMenu.Item></>)}</DropdownMenu.Content>diff --git a/packages/excalidraw/components/App.tsx b/packages/excalidraw/components/App.tsxindex b0293b781b..a5151851b5 100644--- a/packages/excalidraw/components/App.tsx+++ b/packages/excalidraw/components/App.tsx@@ -83,7 +83,6 @@ import {ZOOM_STEP,POINTER_EVENTS,TOOL_TYPE,- EDITOR_LS_KEYS,isIOS,supportsResizeObserver,DEFAULT_COLLISION_THRESHOLD,@@ -183,6 +182,7 @@ import type {ExcalidrawIframeElement,ExcalidrawEmbeddableElement,Ordered,+ MagicGenerationData,ExcalidrawNonSelectionElement,ExcalidrawArrowElement,} from "../element/types";@@ -257,6 +257,7 @@ import type {UnsubscribeCallback,EmbedsValidationStatus,ElementsPendingErasure,+ GenerateDiagramToCode,NullableGridSize,} from "../types";import {@@ -405,13 +406,9 @@ import {} from "../cursor";import { Emitter } from "../emitter";import { ElementCanvasButtons } from "../element/ElementCanvasButtons";-import type { MagicCacheData } from "../data/magic";-import { diagramToHTML } from "../data/magic";-import { exportToBlob } from "../../utils/export";import { COLOR_PALETTE } from "../colors";import { ElementCanvasButton } from "./MagicButton";import { MagicIcon, copyIcon, fullscreenIcon } from "./icons";-import { EditorLocalStorage } from "../data/EditorLocalStorage";import FollowMode from "./FollowMode/FollowMode";import { Store, StoreAction } from "../store";import { AnimationFrameHandler } from "../animation-frame-handler";@@ -1018,7 +1015,7 @@ class App extends React.Component<AppProps, AppState> {if (isIframeElement(el)) {src = null;- const data: MagicCacheData = (el.customData?.generationData ??+ const data: MagicGenerationData = (el.customData?.generationData ??this.magicGenerations.get(el.id)) || {status: "error",message: "No generation data",@@ -1559,10 +1556,6 @@ class App extends React.Component<AppProps, AppState> {}app={this}isCollaborating={this.props.isCollaborating}- openAIKey={this.OPENAI_KEY}- isOpenAIKeyPersisted={this.OPENAI_KEY_IS_PERSISTED}- onOpenAIAPIKeyChange={this.onOpenAIKeyChange}- onMagicSettingsConfirm={this.onMagicSettingsConfirm}>{this.props.children}</LayerUI>@@ -1807,7 +1800,7 @@ class App extends React.Component<AppProps, AppState> {private magicGenerations = new Map<ExcalidrawIframeElement["id"],- MagicCacheData+ MagicGenerationData>();private updateMagicGeneration = ({@@ -1815,7 +1808,7 @@ class App extends React.Component<AppProps, AppState> {data,}: {frameElement: ExcalidrawIframeElement;- data: MagicCacheData;+ data: MagicGenerationData;}) => {if (data.status === "pending") {// We don't wanna persist pending state to storage. It should be in-app@@ -1838,31 +1831,26 @@ class App extends React.Component<AppProps, AppState> {this.triggerRender();};- private getTextFromElements(elements: readonly ExcalidrawElement[]) {- const text = elements- .reduce((acc: string[], element) => {- if (isTextElement(element)) {- acc.push(element.text);- }- return acc;- }, [])- .join("\n\n");- return text;+ public plugins: {+ diagramToCode?: {+ generate: GenerateDiagramToCode;+ };+ } = {};++ public setPlugins(plugins: Partial<App["plugins"]>) {+ Object.assign(this.plugins, plugins);}private async onMagicFrameGenerate(magicFrame: ExcalidrawMagicFrameElement,source: "button" | "upstream",) {- if (!this.OPENAI_KEY) {+ const generateDiagramToCode = this.plugins.diagramToCode?.generate;++ if (!generateDiagramToCode) {this.setState({- openDialog: {- name: "settings",- tab: "diagram-to-code",- source: "generation",- },+ errorMessage: "No diagram to code plugin found",});- trackEvent("ai", "generate (missing key)", "d2c");return;}@@ -1901,68 +1889,50 @@ class App extends React.Component<AppProps, AppState> {selectedElementIds: { [frameElement.id]: true },});- const blob = await exportToBlob({- elements: this.scene.getNonDeletedElements(),- appState: {- ...this.state,- exportBackground: true,- viewBackgroundColor: this.state.viewBackgroundColor,- },- exportingFrame: magicFrame,- files: this.files,- });-- const dataURL = await getDataURL(blob);+ trackEvent("ai", "generate (start)", "d2c");+ try {+ const { html } = await generateDiagramToCode({+ frame: magicFrame,+ children: magicFrameChildren,+ });- const textFromFrameChildren = this.getTextFromElements(magicFrameChildren);+ trackEvent("ai", "generate (success)", "d2c");- trackEvent("ai", "generate (start)", "d2c");+ if (!html.trim()) {+ this.updateMagicGeneration({+ frameElement,+ data: {+ status: "error",+ code: "ERR_OAI",+ message: "Nothing genereated :(",+ },+ });+ return;+ }- const result = await diagramToHTML({- image: dataURL,- apiKey: this.OPENAI_KEY,- text: textFromFrameChildren,- theme: this.state.theme,- });+ const parsedHtml =+ html.includes("<!DOCTYPE html>") && html.includes("</html>")+ ? html.slice(+ html.indexOf("<!DOCTYPE html>"),+ html.indexOf("</html>") + "</html>".length,+ )+ : html;- if (!result.ok) {- trackEvent("ai", "generate (failed)", "d2c");- console.error(result.error);this.updateMagicGeneration({frameElement,- data: {- status: "error",- code: "ERR_OAI",- message: result.error?.message || "Unknown error during generation",- },+ data: { status: "done", html: parsedHtml },});- return;- }- trackEvent("ai", "generate (success)", "d2c");-- if (result.choices[0].message.content == null) {+ } catch (error: any) {+ trackEvent("ai", "generate (failed)", "d2c");this.updateMagicGeneration({frameElement,data: {status: "error",code: "ERR_OAI",- message: "Nothing genereated :(",+ message: error.message || "Unknown error during generation",},});- return;}-- const message = result.choices[0].message.content;-- const html = message.slice(- message.indexOf("<!DOCTYPE html>"),- message.indexOf("</html>") + "</html>".length,- );-- this.updateMagicGeneration({- frameElement,- data: { status: "done", html },- });}private onIframeSrcCopy(element: ExcalidrawIframeElement) {@@ -1976,70 +1946,7 @@ class App extends React.Component<AppProps, AppState> {}}- private OPENAI_KEY: string | null = EditorLocalStorage.get(- EDITOR_LS_KEYS.OAI_API_KEY,- );- private OPENAI_KEY_IS_PERSISTED: boolean =- EditorLocalStorage.has(EDITOR_LS_KEYS.OAI_API_KEY) || false;-- private onOpenAIKeyChange = (- openAIKey: string | null,- shouldPersist: boolean,- ) => {- this.OPENAI_KEY = openAIKey || null;- if (shouldPersist) {- const didPersist = EditorLocalStorage.set(- EDITOR_LS_KEYS.OAI_API_KEY,- openAIKey,- );- this.OPENAI_KEY_IS_PERSISTED = didPersist;- } else {- this.OPENAI_KEY_IS_PERSISTED = false;- }- };-- private onMagicSettingsConfirm = (- apiKey: string,- shouldPersist: boolean,- source: "tool" | "generation" | "settings",- ) => {- this.OPENAI_KEY = apiKey || null;- this.onOpenAIKeyChange(this.OPENAI_KEY, shouldPersist);-- if (source === "settings") {- return;- }-- const selectedElements = this.scene.getSelectedElements({- selectedElementIds: this.state.selectedElementIds,- });-- if (apiKey) {- if (selectedElements.length) {- this.onMagicframeToolSelect();- } else {- this.setActiveTool({ type: "magicframe" });- }- } else if (!isMagicFrameElement(selectedElements[0])) {- // even if user didn't end up setting api key, let's pick the tool- // so they can draw up a frame and move forward- this.setActiveTool({ type: "magicframe" });- }- };-public onMagicframeToolSelect = () => {- if (!this.OPENAI_KEY) {- this.setState({- openDialog: {- name: "settings",- tab: "diagram-to-code",- source: "tool",- },- });- trackEvent("ai", "tool-select (missing key)", "d2c");- return;- }-const selectedElements = this.scene.getSelectedElements({selectedElementIds: this.state.selectedElementIds,});diff --git a/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx b/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsxnew file mode 100644index 0000000000..9505999635--- /dev/null+++ b/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx@@ -0,0 +1,17 @@+import { useLayoutEffect } from "react";+import { useApp } from "../App";+import type { GenerateDiagramToCode } from "../../types";++export const DiagramToCodePlugin = (props: {+ generate: GenerateDiagramToCode;+}) => {+ const app = useApp();++ useLayoutEffect(() => {+ app.setPlugins({+ diagramToCode: { generate: props.generate },+ });+ }, [app, props.generate]);++ return null;+};diff --git a/packages/excalidraw/components/LayerUI.tsx b/packages/excalidraw/components/LayerUI.tsxindex cb641bb619..3353cd5220 100644--- a/packages/excalidraw/components/LayerUI.tsx+++ b/packages/excalidraw/components/LayerUI.tsx@@ -60,7 +60,6 @@ import { mutateElement } from "../element/mutateElement";import { ShapeCache } from "../scene/ShapeCache";import Scene from "../scene/Scene";import { LaserPointerButton } from "./LaserPointerButton";-import { MagicSettings } from "./MagicSettings";import { TTDDialog } from "./TTDDialog/TTDDialog";import { Stats } from "./Stats";import { actionToggleStats } from "../actions";@@ -85,14 +84,6 @@ interface LayerUIProps {children?: React.ReactNode;app: AppClassProperties;isCollaborating: boolean;- openAIKey: string | null;- isOpenAIKeyPersisted: boolean;- onOpenAIAPIKeyChange: (apiKey: string, shouldPersist: boolean) => void;- onMagicSettingsConfirm: (- apiKey: string,- shouldPersist: boolean,- source: "tool" | "generation" | "settings",- ) => void;}const DefaultMainMenu: React.FC<{@@ -149,10 +140,6 @@ const LayerUI = ({children,app,isCollaborating,- openAIKey,- isOpenAIKeyPersisted,- onOpenAIAPIKeyChange,- onMagicSettingsConfirm,}: LayerUIProps) => {const device = useDevice();const tunnels = useInitializeTunnels();@@ -482,25 +469,6 @@ const LayerUI = ({}}/>)}- {appState.openDialog?.name === "settings" && (- <MagicSettings- openAIKey={openAIKey}- isPersisted={isOpenAIKeyPersisted}- onChange={onOpenAIAPIKeyChange}- onConfirm={(apiKey, shouldPersist) => {- const source =- appState.openDialog?.name === "settings"- ? appState.openDialog?.source- : "settings";- setAppState({ openDialog: null }, () => {- onMagicSettingsConfirm(apiKey, shouldPersist, source);- });- }}- onClose={() => {- setAppState({ openDialog: null });- }}- />- )}<ActiveConfirmDialog /><tunnels.OverwriteConfirmDialogTunnel.Out />{renderImageExportDialog()}diff --git a/packages/excalidraw/components/MagicSettings.scss b/packages/excalidraw/components/MagicSettings.scssdeleted file mode 100644index bd07d84003..0000000000--- a/packages/excalidraw/components/MagicSettings.scss+++ /dev/null@@ -1,18 +0,0 @@-.excalidraw {- .MagicSettings {- .Island {- height: 100%;- display: flex;- flex-direction: column;- }- }-- .MagicSettings-confirm {- padding: 0.5rem 1rem;- }-- .MagicSettings__confirm {- margin-top: 2rem;- margin-right: auto;- }-}diff --git a/packages/excalidraw/components/MagicSettings.tsx b/packages/excalidraw/components/MagicSettings.tsxdeleted file mode 100644index 855ab109d7..0000000000--- a/packages/excalidraw/components/MagicSettings.tsx+++ /dev/null@@ -1,160 +0,0 @@-import { useState } from "react";-import { Dialog } from "./Dialog";-import { TextField } from "./TextField";-import { MagicIcon, OpenAIIcon } from "./icons";-import { FilledButton } from "./FilledButton";-import { CheckboxItem } from "./CheckboxItem";-import { KEYS } from "../keys";-import { useUIAppState } from "../context/ui-appState";-import { InlineIcon } from "./InlineIcon";-import { Paragraph } from "./Paragraph";--import "./MagicSettings.scss";-import TTDDialogTabs from "./TTDDialog/TTDDialogTabs";-import { TTDDialogTab } from "./TTDDialog/TTDDialogTab";--export const MagicSettings = (props: {- openAIKey: string | null;- isPersisted: boolean;- onChange: (key: string, shouldPersist: boolean) => void;- onConfirm: (key: string, shouldPersist: boolean) => void;- onClose: () => void;-}) => {- const [keyInputValue, setKeyInputValue] = useState(props.openAIKey || "");- const [shouldPersist, setShouldPersist] = useState<boolean>(- props.isPersisted,- );-- const appState = useUIAppState();-- const onConfirm = () => {- props.onConfirm(keyInputValue.trim(), shouldPersist);- };-- if (appState.openDialog?.name !== "settings") {- return null;- }-- return (- <Dialog- onCloseRequest={() => {- props.onClose();- props.onConfirm(keyInputValue.trim(), shouldPersist);- }}- title={- <div style={{ display: "flex" }}>- Wireframe to Code (AI){" "}- <div- style={{- display: "flex",- alignItems: "center",- justifyContent: "center",- padding: "0.1rem 0.5rem",- marginLeft: "1rem",- fontSize: 14,- borderRadius: "12px",- background: "var(--color-promo)",- color: "var(--color-surface-lowest)",- }}- >- Experimental- </div>- </div>- }- className="MagicSettings"- autofocus={false}- >- {/* <h2- style={{- margin: 0,- fontSize: "1.25rem",- paddingLeft: "2.5rem",- }}- >- AI Settings- </h2> */}- <TTDDialogTabs dialog="settings" tab={appState.openDialog.tab}>- {/* <TTDDialogTabTriggers>- <TTDDialogTabTrigger tab="text-to-diagram">- <InlineIcon icon={brainIcon} /> Text to diagram- </TTDDialogTabTrigger>- <TTDDialogTabTrigger tab="diagram-to-code">- <InlineIcon icon={MagicIcon} /> Wireframe to code- </TTDDialogTabTrigger>- </TTDDialogTabTriggers> */}- {/* <TTDDialogTab className="ttd-dialog-content" tab="text-to-diagram">- TODO- </TTDDialogTab> */}- <TTDDialogTab- // className="ttd-dialog-content"- tab="diagram-to-code"- >- <Paragraph>- For the diagram-to-code feature we use{" "}- <InlineIcon icon={OpenAIIcon} />- OpenAI.- </Paragraph>- <Paragraph>- While the OpenAI API is in beta, its use is strictly limited — as- such we require you use your own API key. You can create an{" "}- <a- href="https://platform.openai.com/login?launch"- rel="noopener noreferrer"- target="_blank"- >- OpenAI account- </a>- , add a small credit (5 USD minimum), and{" "}- <a- href="https://platform.openai.com/api-keys"- rel="noopener noreferrer"- target="_blank"- >- generate your own API key- </a>- .- </Paragraph>- <Paragraph>- Your OpenAI key does not leave the browser, and you can also set- your own limit in your OpenAI account dashboard if needed.- </Paragraph>- <TextField- isRedacted- value={keyInputValue}- placeholder="Paste your API key here"- label="OpenAI API key"- onChange={(value) => {- setKeyInputValue(value);- props.onChange(value.trim(), shouldPersist);- }}- selectOnRender- onKeyDown={(event) => event.key === KEYS.ENTER && onConfirm()}- />- <Paragraph>- By default, your API token is not persisted anywhere so you'll need- to insert it again after reload. But, you can persist locally in- your browser below.- </Paragraph>-- <CheckboxItem checked={shouldPersist} onChange={setShouldPersist}>- Persist API key in browser storage- </CheckboxItem>-- <Paragraph>- Once API key is set, you can use the <InlineIcon icon={MagicIcon} />{" "}- tool to wrap your elements in a frame that will then allow you to- turn it into code. This dialog can be accessed using the{" "}- <b>AI Settings</b> <InlineIcon icon={OpenAIIcon} />.- </Paragraph>-- <FilledButton- className="MagicSettings__confirm"- size="large"- label="Confirm"- onClick={onConfirm}- />- </TTDDialogTab>- </TTDDialogTabs>- </Dialog>- );-};diff --git a/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx b/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsxindex 30add91e59..439f92844d 100644--- a/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx+++ b/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx@@ -7,10 +7,7 @@ import { isMemberOf } from "../../utils";const TTDDialogTabs = (props: {children: ReactNode;- } & (- | { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }- | { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" }- ),+ } & { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" },) => {const setAppState = useExcalidrawSetAppState();@@ -39,13 +36,6 @@ const TTDDialogTabs = (}}if (- props.dialog === "settings" &&- isMemberOf(["text-to-diagram", "diagram-to-code"], tab)- ) {- setAppState({- openDialog: { name: props.dialog, tab, source: "settings" },- });- } else if (props.dialog === "ttd" &&isMemberOf(["text-to-diagram", "mermaid"], tab)) {diff --git a/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx b/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsxindex 0dacaebf98..171c1c71e7 100644--- a/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx+++ b/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx@@ -13,6 +13,7 @@ const DropdownMenuItemLink = ({onSelect,className = "",selected,+ rel = "noreferrer",...rest}: {href: string;@@ -22,6 +23,7 @@ const DropdownMenuItemLink = ({className?: string;selected?: boolean;onSelect?: (event: Event) => void;+ rel?: string;} & React.AnchorHTMLAttributes<HTMLAnchorElement>) => {const handleClick = useHandleDropdownMenuItemClick(rest.onClick, onSelect);diff --git a/packages/excalidraw/data/magic.ts b/packages/excalidraw/data/magic.tsdeleted file mode 100644index 883a3bdb90..0000000000--- a/packages/excalidraw/data/magic.ts+++ /dev/null@@ -1,105 +0,0 @@-import { THEME } from "../constants";-import type { Theme } from "../element/types";-import type { DataURL } from "../types";-import type { OpenAIInput, OpenAIOutput } from "./ai/types";--export type MagicCacheData =- | {- status: "pending";- }- | { status: "done"; html: string }- | {- status: "error";- message?: string;- code: "ERR_GENERATION_INTERRUPTED" | string;- };--const SYSTEM_PROMPT = `You are a skilled front-end developer who builds interactive prototypes from wireframes, and is an expert at CSS Grid and Flex design.-Your role is to transform low-fidelity wireframes into working front-end HTML code.--YOU MUST FOLLOW FOLLOWING RULES:--- Use HTML, CSS, JavaScript to build a responsive, accessible, polished prototype-- Leverage Tailwind for styling and layout (import as script <script src="https://cdn.tailwindcss.com"></script>)-- Inline JavaScript when needed-- Fetch dependencies from CDNs when needed (using unpkg or skypack)-- Source images from Unsplash or create applicable placeholders-- Interpret annotations as intended vs literal UI-- Fill gaps using your expertise in UX and business logic-- generate primarily for desktop UI, but make it responsive.-- Use grid and flexbox wherever applicable.-- Convert the wireframe in its entirety, don't omit elements if possible.--If the wireframes, diagrams, or text is unclear or unreadable, refer to provided text for clarification.--Your goal is a production-ready prototype that brings the wireframes to life.--Please output JUST THE HTML file containing your best attempt at implementing the provided wireframes.`;--export async function diagramToHTML({- image,- apiKey,- text,- theme = THEME.LIGHT,-}: {- image: DataURL;- apiKey: string;- text: string;- theme?: Theme;-}) {- const body: OpenAIInput.ChatCompletionCreateParamsBase = {- model: "gpt-4-vision-preview",- // 4096 are max output tokens allowed for `gpt-4-vision-preview` currently- max_tokens: 4096,- temperature: 0.1,- messages: [- {- role: "system",- content: SYSTEM_PROMPT,- },- {- role: "user",- content: [- {- type: "image_url",- image_url: {- url: image,- detail: "high",- },- },- {- type: "text",- text: `Above is the reference wireframe. Please make a new website based on these and return just the HTML file. Also, please make it for the ${theme} theme. What follows are the wireframe's text annotations (if any)...`,- },- {- type: "text",- text,- },- ],- },- ],- };-- let result:- | ({ ok: true } & OpenAIOutput.ChatCompletion)- | ({ ok: false } & OpenAIOutput.APIError);-- const resp = await fetch("https://api.openai.com/v1/chat/completions", {- method: "POST",- headers: {- "Content-Type": "application/json",- Authorization: `Bearer ${apiKey}`,- },- body: JSON.stringify(body),- });-- if (resp.ok) {- const json: OpenAIOutput.ChatCompletion = await resp.json();- result = { ...json, ok: true };- } else {- const json: OpenAIOutput.APIError = await resp.json();- result = { ...json, ok: false };- }-- return result;-}diff --git a/packages/excalidraw/element/index.ts b/packages/excalidraw/element/index.tsindex d6bc8f6156..a9b747681f 100644--- a/packages/excalidraw/element/index.ts+++ b/packages/excalidraw/element/index.ts@@ -46,7 +46,7 @@ export {dragNewElement,} from "./dragElements";export { isTextElement, isExcalidrawElement } from "./typeChecks";-export { redrawTextBoundingBox } from "./textElement";+export { redrawTextBoundingBox, getTextFromElements } from "./textElement";export {getPerfectElementSize,getLockedLinearCursorAlignSize,diff --git a/packages/excalidraw/element/textElement.ts b/packages/excalidraw/element/textElement.tsindex 696f809e6b..9fb4766b6e 100644--- a/packages/excalidraw/element/textElement.ts+++ b/packages/excalidraw/element/textElement.ts@@ -886,3 +886,19 @@ export const getMinTextElementWidth = () => {return measureText("", font, lineHeight).width + BOUND_TEXT_PADDING * 2;};++/** retrieves text from text elements and concatenates to a single string */+export const getTextFromElements = (+ elements: readonly ExcalidrawElement[],+ separator = "\n\n",+) => {+ const text = elements+ .reduce((acc: string[], element) => {+ if (isTextElement(element)) {+ acc.push(element.text);+ }+ return acc;+ }, [])+ .join(separator);+ return text;+};diff --git a/packages/excalidraw/element/types.ts b/packages/excalidraw/element/types.tsindex 33372764b7..4322cf8510 100644--- a/packages/excalidraw/element/types.ts+++ b/packages/excalidraw/element/types.ts@@ -12,7 +12,6 @@ import type {Merge,ValueOf,} from "../utility-types";-import type { MagicCacheData } from "../data/magic";export type ChartType = "bar" | "line";export type FillStyle = "hachure" | "cross-hatch" | "solid" | "zigzag";@@ -101,11 +100,22 @@ export type ExcalidrawEmbeddableElement = _ExcalidrawElementBase &type: "embeddable";}>;+export type MagicGenerationData =+ | {+ status: "pending";+ }+ | { status: "done"; html: string }+ | {+ status: "error";+ message?: string;+ code: "ERR_GENERATION_INTERRUPTED" | string;+ };+export type ExcalidrawIframeElement = _ExcalidrawElementBase &Readonly<{type: "iframe";// TODO move later to AI-specific frame- customData?: { generationData?: MagicCacheData };+ customData?: { generationData?: MagicGenerationData };}>;export type ExcalidrawIframeLikeElement =diff --git a/packages/excalidraw/index.tsx b/packages/excalidraw/index.tsxindex ce8bdbe876..2e34eb7a6d 100644--- a/packages/excalidraw/index.tsx+++ b/packages/excalidraw/index.tsx@@ -213,6 +213,7 @@ export {hashString,isInvisiblySmallElement,getNonDeletedElements,+ getTextFromElements,} from "./element";export { defaultLang, useI18n, languages } from "./i18n";export {@@ -287,3 +288,6 @@ export {isElementInsideBBox,elementPartiallyOverlapsWithOrContainsBBox,} from "../utils/withinBounds";++export { DiagramToCodePlugin } from "./components/DiagramToCodePlugin/DiagramToCodePlugin";+export { getDataURL } from "./data/blob";diff --git a/packages/excalidraw/locales/en.json b/packages/excalidraw/locales/en.jsonindex b67624efe5..882293c8a8 100644--- a/packages/excalidraw/locales/en.json+++ b/packages/excalidraw/locales/en.json@@ -272,8 +272,7 @@"laser": "Laser pointer","hand": "Hand (panning tool)","extraTools": "More tools",- "mermaidToExcalidraw": "Mermaid to Excalidraw",- "magicSettings": "AI settings"+ "mermaidToExcalidraw": "Mermaid to Excalidraw"},"element": {"rectangle": "Rectangle",diff --git a/packages/excalidraw/types.ts b/packages/excalidraw/types.tsindex 3b107f26e2..0721f932dd 100644--- a/packages/excalidraw/types.ts+++ b/packages/excalidraw/types.ts@@ -325,14 +325,6 @@ export interface AppState {openDialog:| null| { name: "imageExport" | "help" | "jsonExport" }- | {- name: "settings";- source:- | "tool" // when magicframe tool is selected- | "generation" // when magicframe generate button is clicked- | "settings"; // when AI settings dialog is explicitly invoked- tab: "text-to-diagram" | "diagram-to-code";- }| { name: "ttd"; tab: "text-to-diagram" | "mermaid" }| { name: "commandPalette" };/**@@ -655,6 +647,8 @@ export type AppClassProperties = {dismissLinearEditor: App["dismissLinearEditor"];flowChartCreator: App["flowChartCreator"];getEffectiveGridSize: App["getEffectiveGridSize"];+ setPlugins: App["setPlugins"];+ plugins: App["plugins"];};export type PointerDownState = Readonly<{@@ -842,3 +836,8 @@ export type PendingExcalidrawElements = ExcalidrawElement[];export type NullableGridSize =| (AppState["gridSize"] & MakeBrand<"NullableGridSize">)| null;++export type GenerateDiagramToCode = (props: {+ frame: ExcalidrawMagicFrameElement;+ children: readonly ExcalidrawElement[];+}) => MaybePromise<{ html: string }>;diff --git a/packages/excalidraw/utils.ts b/packages/excalidraw/utils.tsindex 3216eba5c2..0f3d11f717 100644--- a/packages/excalidraw/utils.ts+++ b/packages/excalidraw/utils.ts@@ -1166,3 +1166,11 @@ export const promiseTry = async <TValue, TArgs extends unknown[]>(export const isAnyTrue = (...args: boolean[]): boolean =>Math.max(...args.map((arg) => (arg ? 1 : 0))) > 0;++export const safelyParseJSON = (json: string): Record<string, any> | null => {+ try {+ return JSON.parse(json);+ } catch {+ return null;+ }+};