Skip to content

Virtual whiteboard for sketching hand-drawn like diagrams

feat: rewrite d2c to not require token (#8269)

David Luzarb5d7f5b4ba85
diff --git a/excalidraw-app/App.tsx b/excalidraw-app/App.tsx
index a11ea59b32..69d6211293 100644
--- a/excalidraw-app/App.tsx
+++ b/excalidraw-app/App.tsx
@@ -22,7 +22,6 @@ import { t } from "../packages/excalidraw/i18n";
import {
Excalidraw,
LiveCollaborationTrigger,
- TTDDialog,
TTDDialogTrigger,
StoreAction,
reconcileElements,
@@ -121,6 +120,7 @@ import {
import { appThemeAtom, useHandleAppTheme } from "./useHandleAppTheme";
import { getPreferredLanguage } from "./app-language/language-detector";
import { useAppLangCode } from "./app-language/language-state";
+import { AIComponents } from "./components/AI";
polyfill();
@@ -846,63 +846,8 @@ const ExcalidrawWrapper = () => {
)}
</OverwriteConfirmDialog>
<AppFooter />
- <TTDDialog
- onTextSubmit={async (input) => {
- try {
- const response = await fetch(
- `${
- import.meta.env.VITE_APP_AI_BACKEND
- }/v1/ai/text-to-diagram/generate`,
- {
- method: "POST",
- headers: {
- Accept: "application/json",
- "Content-Type": "application/json",
- },
- body: JSON.stringify({ prompt: input }),
- },
- );
-
- const rateLimit = response.headers.has("X-Ratelimit-Limit")
- ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)
- : undefined;
-
- const rateLimitRemaining = response.headers.has(
- "X-Ratelimit-Remaining",
- )
- ? parseInt(
- response.headers.get("X-Ratelimit-Remaining") || "0",
- 10,
- )
- : undefined;
-
- const json = await response.json();
-
- if (!response.ok) {
- if (response.status === 429) {
- return {
- rateLimit,
- rateLimitRemaining,
- error: new Error(
- "Too many requests today, please try again tomorrow!",
- ),
- };
- }
+ {excalidrawAPI && <AIComponents excalidrawAPI={excalidrawAPI} />}
- throw new Error(json.message || "Generation failed...");
- }
-
- const generatedResponse = json.generatedResponse;
- if (!generatedResponse) {
- throw new Error("Generation failed...");
- }
-
- return { generatedResponse, rateLimit, rateLimitRemaining };
- } catch (err: any) {
- throw new Error("Request failed");
- }
- }}
- />
<TTDDialogTrigger />
{isCollaborating && isOffline && (
<div className="collab-offline-warning">
diff --git a/excalidraw-app/components/AI.tsx b/excalidraw-app/components/AI.tsx
new file mode 100644
index 0000000000..621d6befa7
--- /dev/null
+++ b/excalidraw-app/components/AI.tsx
@@ -0,0 +1,159 @@
+import type { ExcalidrawImperativeAPI } from "../../packages/excalidraw/types";
+import {
+ DiagramToCodePlugin,
+ exportToBlob,
+ getTextFromElements,
+ MIME_TYPES,
+ TTDDialog,
+} from "../../packages/excalidraw";
+import { getDataURL } from "../../packages/excalidraw/data/blob";
+import { safelyParseJSON } from "../../packages/excalidraw/utils";
+
+export const AIComponents = ({
+ excalidrawAPI,
+}: {
+ excalidrawAPI: ExcalidrawImperativeAPI;
+}) => {
+ return (
+ <>
+ <DiagramToCodePlugin
+ generate={async ({ frame, children }) => {
+ const appState = excalidrawAPI.getAppState();
+
+ const blob = await exportToBlob({
+ elements: children,
+ appState: {
+ ...appState,
+ exportBackground: true,
+ viewBackgroundColor: appState.viewBackgroundColor,
+ },
+ exportingFrame: frame,
+ files: excalidrawAPI.getFiles(),
+ mimeType: MIME_TYPES.jpg,
+ });
+
+ const dataURL = await getDataURL(blob);
+
+ const textFromFrameChildren = getTextFromElements(children);
+
+ const response = await fetch(
+ `${
+ import.meta.env.VITE_APP_AI_BACKEND
+ }/v1/ai/diagram-to-code/generate`,
+ {
+ method: "POST",
+ headers: {
+ Accept: "application/json",
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ texts: textFromFrameChildren,
+ image: dataURL,
+ theme: appState.theme,
+ }),
+ },
+ );
+
+ if (!response.ok) {
+ const text = await response.text();
+ const errorJSON = safelyParseJSON(text);
+
+ if (!errorJSON) {
+ throw new Error(text);
+ }
+
+ if (errorJSON.statusCode === 429) {
+ return {
+ html: `<html>
+ <body style="margin: 0; text-align: center">
+ <div style="display: flex; align-items: center; justify-content: center; flex-direction: column; height: 100vh; padding: 0 60px">
+ <div style="color:red">Too many requests today,</br>please try again tomorrow!</div>
+ </br>
+ </br>
+ <div>You can also try <a href="${
+ import.meta.env.VITE_APP_PLUS_LP
+ }/plus?utm_source=excalidraw&utm_medium=app&utm_content=d2c" target="_blank" rel="noreferrer noopener">Excalidraw+</a> to get more requests.</div>
+ </div>
+ </body>
+ </html>`,
+ };
+ }
+
+ throw new Error(errorJSON.message || text);
+ }
+
+ try {
+ const { html } = await response.json();
+
+ if (!html) {
+ throw new Error("Generation failed (invalid response)");
+ }
+ return {
+ html,
+ };
+ } catch (error: any) {
+ throw new Error("Generation failed (invalid response)");
+ }
+ }}
+ />
+
+ <TTDDialog
+ onTextSubmit={async (input) => {
+ try {
+ const response = await fetch(
+ `${
+ import.meta.env.VITE_APP_AI_BACKEND
+ }/v1/ai/text-to-diagram/generate`,
+ {
+ method: "POST",
+ headers: {
+ Accept: "application/json",
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({ prompt: input }),
+ },
+ );
+
+ const rateLimit = response.headers.has("X-Ratelimit-Limit")
+ ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)
+ : undefined;
+
+ const rateLimitRemaining = response.headers.has(
+ "X-Ratelimit-Remaining",
+ )
+ ? parseInt(
+ response.headers.get("X-Ratelimit-Remaining") || "0",
+ 10,
+ )
+ : undefined;
+
+ const json = await response.json();
+
+ if (!response.ok) {
+ if (response.status === 429) {
+ return {
+ rateLimit,
+ rateLimitRemaining,
+ error: new Error(
+ "Too many requests today, please try again tomorrow!",
+ ),
+ };
+ }
+
+ throw new Error(json.message || "Generation failed...");
+ }
+
+ const generatedResponse = json.generatedResponse;
+ if (!generatedResponse) {
+ throw new Error("Generation failed...");
+ }
+
+ return { generatedResponse, rateLimit, rateLimitRemaining };
+ } catch (err: any) {
+ throw new Error("Request failed");
+ }
+ }}
+ />
+ </>
+ );
+};
diff --git a/packages/excalidraw/actions/actionClipboard.tsx b/packages/excalidraw/actions/actionClipboard.tsx
index e4f998d016..4d34bf968c 100644
--- a/packages/excalidraw/actions/actionClipboard.tsx
+++ b/packages/excalidraw/actions/actionClipboard.tsx
@@ -10,7 +10,7 @@ import {
} from "../clipboard";
import { actionDeleteSelected } from "./actionDeleteSelected";
import { exportCanvas, prepareElementsForExport } from "../data/index";
-import { isTextElement } from "../element";
+import { getTextFromElements, isTextElement } from "../element";
import { t } from "../i18n";
import { isFirefox } from "../constants";
import { DuplicateIcon, cutIcon, pngIcon, svgIcon } from "../components/icons";
@@ -239,16 +239,8 @@ export const copyText = register({
includeBoundTextElement: true,
});
- const text = selectedElements
- .reduce((acc: string[], element) => {
- if (isTextElement(element)) {
- acc.push(element.text);
- }
- return acc;
- }, [])
- .join("\n\n");
try {
- copyTextToSystemClipboard(text);
+ copyTextToSystemClipboard(getTextFromElements(selectedElements));
} catch (e) {
throw new Error(t("errors.copyToSystemClipboardFailed"));
}
diff --git a/packages/excalidraw/components/Actions.tsx b/packages/excalidraw/components/Actions.tsx
index 91102aef2c..15bddefdd5 100644
--- a/packages/excalidraw/components/Actions.tsx
+++ b/packages/excalidraw/components/Actions.tsx
@@ -45,7 +45,6 @@ import {
frameToolIcon,
mermaidLogoIcon,
laserPointerToolIcon,
- OpenAIIcon,
MagicIcon,
} from "./icons";
import { KEYS } from "../keys";
@@ -400,7 +399,7 @@ export const ShapesSwitcher = ({
>
{t("toolBar.mermaidToExcalidraw")}
</DropdownMenu.Item>
- {app.props.aiEnabled !== false && (
+ {app.props.aiEnabled !== false && app.plugins.diagramToCode && (
<>
<DropdownMenu.Item
onSelect={() => app.onMagicframeToolSelect()}
@@ -410,20 +409,6 @@ export const ShapesSwitcher = ({
{t("toolBar.magicframe")}
<DropdownMenu.Item.Badge>AI</DropdownMenu.Item.Badge>
</DropdownMenu.Item>
- <DropdownMenu.Item
- onSelect={() => {
- trackEvent("ai", "open-settings", "d2c");
- app.setOpenDialog({
- name: "settings",
- source: "settings",
- tab: "diagram-to-code",
- });
- }}
- icon={OpenAIIcon}
- data-testid="toolbar-magicSettings"
- >
- {t("toolBar.magicSettings")}
- </DropdownMenu.Item>
</>
)}
</DropdownMenu.Content>
diff --git a/packages/excalidraw/components/App.tsx b/packages/excalidraw/components/App.tsx
index b0293b781b..a5151851b5 100644
--- a/packages/excalidraw/components/App.tsx
+++ b/packages/excalidraw/components/App.tsx
@@ -83,7 +83,6 @@ import {
ZOOM_STEP,
POINTER_EVENTS,
TOOL_TYPE,
- EDITOR_LS_KEYS,
isIOS,
supportsResizeObserver,
DEFAULT_COLLISION_THRESHOLD,
@@ -183,6 +182,7 @@ import type {
ExcalidrawIframeElement,
ExcalidrawEmbeddableElement,
Ordered,
+ MagicGenerationData,
ExcalidrawNonSelectionElement,
ExcalidrawArrowElement,
} from "../element/types";
@@ -257,6 +257,7 @@ import type {
UnsubscribeCallback,
EmbedsValidationStatus,
ElementsPendingErasure,
+ GenerateDiagramToCode,
NullableGridSize,
} from "../types";
import {
@@ -405,13 +406,9 @@ import {
} from "../cursor";
import { Emitter } from "../emitter";
import { ElementCanvasButtons } from "../element/ElementCanvasButtons";
-import type { MagicCacheData } from "../data/magic";
-import { diagramToHTML } from "../data/magic";
-import { exportToBlob } from "../../utils/export";
import { COLOR_PALETTE } from "../colors";
import { ElementCanvasButton } from "./MagicButton";
import { MagicIcon, copyIcon, fullscreenIcon } from "./icons";
-import { EditorLocalStorage } from "../data/EditorLocalStorage";
import FollowMode from "./FollowMode/FollowMode";
import { Store, StoreAction } from "../store";
import { AnimationFrameHandler } from "../animation-frame-handler";
@@ -1018,7 +1015,7 @@ class App extends React.Component<AppProps, AppState> {
if (isIframeElement(el)) {
src = null;
- const data: MagicCacheData = (el.customData?.generationData ??
+ const data: MagicGenerationData = (el.customData?.generationData ??
this.magicGenerations.get(el.id)) || {
status: "error",
message: "No generation data",
@@ -1559,10 +1556,6 @@ class App extends React.Component<AppProps, AppState> {
}
app={this}
isCollaborating={this.props.isCollaborating}
- openAIKey={this.OPENAI_KEY}
- isOpenAIKeyPersisted={this.OPENAI_KEY_IS_PERSISTED}
- onOpenAIAPIKeyChange={this.onOpenAIKeyChange}
- onMagicSettingsConfirm={this.onMagicSettingsConfirm}
>
{this.props.children}
</LayerUI>
@@ -1807,7 +1800,7 @@ class App extends React.Component<AppProps, AppState> {
private magicGenerations = new Map<
ExcalidrawIframeElement["id"],
- MagicCacheData
+ MagicGenerationData
>();
private updateMagicGeneration = ({
@@ -1815,7 +1808,7 @@ class App extends React.Component<AppProps, AppState> {
data,
}: {
frameElement: ExcalidrawIframeElement;
- data: MagicCacheData;
+ data: MagicGenerationData;
}) => {
if (data.status === "pending") {
// We don't wanna persist pending state to storage. It should be in-app
@@ -1838,31 +1831,26 @@ class App extends React.Component<AppProps, AppState> {
this.triggerRender();
};
- private getTextFromElements(elements: readonly ExcalidrawElement[]) {
- const text = elements
- .reduce((acc: string[], element) => {
- if (isTextElement(element)) {
- acc.push(element.text);
- }
- return acc;
- }, [])
- .join("\n\n");
- return text;
+ public plugins: {
+ diagramToCode?: {
+ generate: GenerateDiagramToCode;
+ };
+ } = {};
+
+ public setPlugins(plugins: Partial<App["plugins"]>) {
+ Object.assign(this.plugins, plugins);
}
private async onMagicFrameGenerate(
magicFrame: ExcalidrawMagicFrameElement,
source: "button" | "upstream",
) {
- if (!this.OPENAI_KEY) {
+ const generateDiagramToCode = this.plugins.diagramToCode?.generate;
+
+ if (!generateDiagramToCode) {
this.setState({
- openDialog: {
- name: "settings",
- tab: "diagram-to-code",
- source: "generation",
- },
+ errorMessage: "No diagram to code plugin found",
});
- trackEvent("ai", "generate (missing key)", "d2c");
return;
}
@@ -1901,68 +1889,50 @@ class App extends React.Component<AppProps, AppState> {
selectedElementIds: { [frameElement.id]: true },
});
- const blob = await exportToBlob({
- elements: this.scene.getNonDeletedElements(),
- appState: {
- ...this.state,
- exportBackground: true,
- viewBackgroundColor: this.state.viewBackgroundColor,
- },
- exportingFrame: magicFrame,
- files: this.files,
- });
-
- const dataURL = await getDataURL(blob);
+ trackEvent("ai", "generate (start)", "d2c");
+ try {
+ const { html } = await generateDiagramToCode({
+ frame: magicFrame,
+ children: magicFrameChildren,
+ });
- const textFromFrameChildren = this.getTextFromElements(magicFrameChildren);
+ trackEvent("ai", "generate (success)", "d2c");
- trackEvent("ai", "generate (start)", "d2c");
+ if (!html.trim()) {
+ this.updateMagicGeneration({
+ frameElement,
+ data: {
+ status: "error",
+ code: "ERR_OAI",
+ message: "Nothing genereated :(",
+ },
+ });
+ return;
+ }
- const result = await diagramToHTML({
- image: dataURL,
- apiKey: this.OPENAI_KEY,
- text: textFromFrameChildren,
- theme: this.state.theme,
- });
+ const parsedHtml =
+ html.includes("<!DOCTYPE html>") && html.includes("</html>")
+ ? html.slice(
+ html.indexOf("<!DOCTYPE html>"),
+ html.indexOf("</html>") + "</html>".length,
+ )
+ : html;
- if (!result.ok) {
- trackEvent("ai", "generate (failed)", "d2c");
- console.error(result.error);
this.updateMagicGeneration({
frameElement,
- data: {
- status: "error",
- code: "ERR_OAI",
- message: result.error?.message || "Unknown error during generation",
- },
+ data: { status: "done", html: parsedHtml },
});
- return;
- }
- trackEvent("ai", "generate (success)", "d2c");
-
- if (result.choices[0].message.content == null) {
+ } catch (error: any) {
+ trackEvent("ai", "generate (failed)", "d2c");
this.updateMagicGeneration({
frameElement,
data: {
status: "error",
code: "ERR_OAI",
- message: "Nothing genereated :(",
+ message: error.message || "Unknown error during generation",
},
});
- return;
}
-
- const message = result.choices[0].message.content;
-
- const html = message.slice(
- message.indexOf("<!DOCTYPE html>"),
- message.indexOf("</html>") + "</html>".length,
- );
-
- this.updateMagicGeneration({
- frameElement,
- data: { status: "done", html },
- });
}
private onIframeSrcCopy(element: ExcalidrawIframeElement) {
@@ -1976,70 +1946,7 @@ class App extends React.Component<AppProps, AppState> {
}
}
- private OPENAI_KEY: string | null = EditorLocalStorage.get(
- EDITOR_LS_KEYS.OAI_API_KEY,
- );
- private OPENAI_KEY_IS_PERSISTED: boolean =
- EditorLocalStorage.has(EDITOR_LS_KEYS.OAI_API_KEY) || false;
-
- private onOpenAIKeyChange = (
- openAIKey: string | null,
- shouldPersist: boolean,
- ) => {
- this.OPENAI_KEY = openAIKey || null;
- if (shouldPersist) {
- const didPersist = EditorLocalStorage.set(
- EDITOR_LS_KEYS.OAI_API_KEY,
- openAIKey,
- );
- this.OPENAI_KEY_IS_PERSISTED = didPersist;
- } else {
- this.OPENAI_KEY_IS_PERSISTED = false;
- }
- };
-
- private onMagicSettingsConfirm = (
- apiKey: string,
- shouldPersist: boolean,
- source: "tool" | "generation" | "settings",
- ) => {
- this.OPENAI_KEY = apiKey || null;
- this.onOpenAIKeyChange(this.OPENAI_KEY, shouldPersist);
-
- if (source === "settings") {
- return;
- }
-
- const selectedElements = this.scene.getSelectedElements({
- selectedElementIds: this.state.selectedElementIds,
- });
-
- if (apiKey) {
- if (selectedElements.length) {
- this.onMagicframeToolSelect();
- } else {
- this.setActiveTool({ type: "magicframe" });
- }
- } else if (!isMagicFrameElement(selectedElements[0])) {
- // even if user didn't end up setting api key, let's pick the tool
- // so they can draw up a frame and move forward
- this.setActiveTool({ type: "magicframe" });
- }
- };
-
public onMagicframeToolSelect = () => {
- if (!this.OPENAI_KEY) {
- this.setState({
- openDialog: {
- name: "settings",
- tab: "diagram-to-code",
- source: "tool",
- },
- });
- trackEvent("ai", "tool-select (missing key)", "d2c");
- return;
- }
-
const selectedElements = this.scene.getSelectedElements({
selectedElementIds: this.state.selectedElementIds,
});
diff --git a/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx b/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx
new file mode 100644
index 0000000000..9505999635
--- /dev/null
+++ b/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx
@@ -0,0 +1,17 @@
+import { useLayoutEffect } from "react";
+import { useApp } from "../App";
+import type { GenerateDiagramToCode } from "../../types";
+
+export const DiagramToCodePlugin = (props: {
+ generate: GenerateDiagramToCode;
+}) => {
+ const app = useApp();
+
+ useLayoutEffect(() => {
+ app.setPlugins({
+ diagramToCode: { generate: props.generate },
+ });
+ }, [app, props.generate]);
+
+ return null;
+};
diff --git a/packages/excalidraw/components/LayerUI.tsx b/packages/excalidraw/components/LayerUI.tsx
index cb641bb619..3353cd5220 100644
--- a/packages/excalidraw/components/LayerUI.tsx
+++ b/packages/excalidraw/components/LayerUI.tsx
@@ -60,7 +60,6 @@ import { mutateElement } from "../element/mutateElement";
import { ShapeCache } from "../scene/ShapeCache";
import Scene from "../scene/Scene";
import { LaserPointerButton } from "./LaserPointerButton";
-import { MagicSettings } from "./MagicSettings";
import { TTDDialog } from "./TTDDialog/TTDDialog";
import { Stats } from "./Stats";
import { actionToggleStats } from "../actions";
@@ -85,14 +84,6 @@ interface LayerUIProps {
children?: React.ReactNode;
app: AppClassProperties;
isCollaborating: boolean;
- openAIKey: string | null;
- isOpenAIKeyPersisted: boolean;
- onOpenAIAPIKeyChange: (apiKey: string, shouldPersist: boolean) => void;
- onMagicSettingsConfirm: (
- apiKey: string,
- shouldPersist: boolean,
- source: "tool" | "generation" | "settings",
- ) => void;
}
const DefaultMainMenu: React.FC<{
@@ -149,10 +140,6 @@ const LayerUI = ({
children,
app,
isCollaborating,
- openAIKey,
- isOpenAIKeyPersisted,
- onOpenAIAPIKeyChange,
- onMagicSettingsConfirm,
}: LayerUIProps) => {
const device = useDevice();
const tunnels = useInitializeTunnels();
@@ -482,25 +469,6 @@ const LayerUI = ({
}}
/>
)}
- {appState.openDialog?.name === "settings" && (
- <MagicSettings
- openAIKey={openAIKey}
- isPersisted={isOpenAIKeyPersisted}
- onChange={onOpenAIAPIKeyChange}
- onConfirm={(apiKey, shouldPersist) => {
- const source =
- appState.openDialog?.name === "settings"
- ? appState.openDialog?.source
- : "settings";
- setAppState({ openDialog: null }, () => {
- onMagicSettingsConfirm(apiKey, shouldPersist, source);
- });
- }}
- onClose={() => {
- setAppState({ openDialog: null });
- }}
- />
- )}
<ActiveConfirmDialog />
<tunnels.OverwriteConfirmDialogTunnel.Out />
{renderImageExportDialog()}
diff --git a/packages/excalidraw/components/MagicSettings.scss b/packages/excalidraw/components/MagicSettings.scss
deleted file mode 100644
index bd07d84003..0000000000
--- a/packages/excalidraw/components/MagicSettings.scss
+++ /dev/null
@@ -1,18 +0,0 @@
-.excalidraw {
- .MagicSettings {
- .Island {
- height: 100%;
- display: flex;
- flex-direction: column;
- }
- }
-
- .MagicSettings-confirm {
- padding: 0.5rem 1rem;
- }
-
- .MagicSettings__confirm {
- margin-top: 2rem;
- margin-right: auto;
- }
-}
diff --git a/packages/excalidraw/components/MagicSettings.tsx b/packages/excalidraw/components/MagicSettings.tsx
deleted file mode 100644
index 855ab109d7..0000000000
--- a/packages/excalidraw/components/MagicSettings.tsx
+++ /dev/null
@@ -1,160 +0,0 @@
-import { useState } from "react";
-import { Dialog } from "./Dialog";
-import { TextField } from "./TextField";
-import { MagicIcon, OpenAIIcon } from "./icons";
-import { FilledButton } from "./FilledButton";
-import { CheckboxItem } from "./CheckboxItem";
-import { KEYS } from "../keys";
-import { useUIAppState } from "../context/ui-appState";
-import { InlineIcon } from "./InlineIcon";
-import { Paragraph } from "./Paragraph";
-
-import "./MagicSettings.scss";
-import TTDDialogTabs from "./TTDDialog/TTDDialogTabs";
-import { TTDDialogTab } from "./TTDDialog/TTDDialogTab";
-
-export const MagicSettings = (props: {
- openAIKey: string | null;
- isPersisted: boolean;
- onChange: (key: string, shouldPersist: boolean) => void;
- onConfirm: (key: string, shouldPersist: boolean) => void;
- onClose: () => void;
-}) => {
- const [keyInputValue, setKeyInputValue] = useState(props.openAIKey || "");
- const [shouldPersist, setShouldPersist] = useState<boolean>(
- props.isPersisted,
- );
-
- const appState = useUIAppState();
-
- const onConfirm = () => {
- props.onConfirm(keyInputValue.trim(), shouldPersist);
- };
-
- if (appState.openDialog?.name !== "settings") {
- return null;
- }
-
- return (
- <Dialog
- onCloseRequest={() => {
- props.onClose();
- props.onConfirm(keyInputValue.trim(), shouldPersist);
- }}
- title={
- <div style={{ display: "flex" }}>
- Wireframe to Code (AI){" "}
- <div
- style={{
- display: "flex",
- alignItems: "center",
- justifyContent: "center",
- padding: "0.1rem 0.5rem",
- marginLeft: "1rem",
- fontSize: 14,
- borderRadius: "12px",
- background: "var(--color-promo)",
- color: "var(--color-surface-lowest)",
- }}
- >
- Experimental
- </div>
- </div>
- }
- className="MagicSettings"
- autofocus={false}
- >
- {/* <h2
- style={{
- margin: 0,
- fontSize: "1.25rem",
- paddingLeft: "2.5rem",
- }}
- >
- AI Settings
- </h2> */}
- <TTDDialogTabs dialog="settings" tab={appState.openDialog.tab}>
- {/* <TTDDialogTabTriggers>
- <TTDDialogTabTrigger tab="text-to-diagram">
- <InlineIcon icon={brainIcon} /> Text to diagram
- </TTDDialogTabTrigger>
- <TTDDialogTabTrigger tab="diagram-to-code">
- <InlineIcon icon={MagicIcon} /> Wireframe to code
- </TTDDialogTabTrigger>
- </TTDDialogTabTriggers> */}
- {/* <TTDDialogTab className="ttd-dialog-content" tab="text-to-diagram">
- TODO
- </TTDDialogTab> */}
- <TTDDialogTab
- // className="ttd-dialog-content"
- tab="diagram-to-code"
- >
- <Paragraph>
- For the diagram-to-code feature we use{" "}
- <InlineIcon icon={OpenAIIcon} />
- OpenAI.
- </Paragraph>
- <Paragraph>
- While the OpenAI API is in beta, its use is strictly limited — as
- such we require you use your own API key. You can create an{" "}
- <a
- href="https://platform.openai.com/login?launch"
- rel="noopener noreferrer"
- target="_blank"
- >
- OpenAI account
- </a>
- , add a small credit (5 USD minimum), and{" "}
- <a
- href="https://platform.openai.com/api-keys"
- rel="noopener noreferrer"
- target="_blank"
- >
- generate your own API key
- </a>
- .
- </Paragraph>
- <Paragraph>
- Your OpenAI key does not leave the browser, and you can also set
- your own limit in your OpenAI account dashboard if needed.
- </Paragraph>
- <TextField
- isRedacted
- value={keyInputValue}
- placeholder="Paste your API key here"
- label="OpenAI API key"
- onChange={(value) => {
- setKeyInputValue(value);
- props.onChange(value.trim(), shouldPersist);
- }}
- selectOnRender
- onKeyDown={(event) => event.key === KEYS.ENTER && onConfirm()}
- />
- <Paragraph>
- By default, your API token is not persisted anywhere so you'll need
- to insert it again after reload. But, you can persist locally in
- your browser below.
- </Paragraph>
-
- <CheckboxItem checked={shouldPersist} onChange={setShouldPersist}>
- Persist API key in browser storage
- </CheckboxItem>
-
- <Paragraph>
- Once API key is set, you can use the <InlineIcon icon={MagicIcon} />{" "}
- tool to wrap your elements in a frame that will then allow you to
- turn it into code. This dialog can be accessed using the{" "}
- <b>AI Settings</b> <InlineIcon icon={OpenAIIcon} />.
- </Paragraph>
-
- <FilledButton
- className="MagicSettings__confirm"
- size="large"
- label="Confirm"
- onClick={onConfirm}
- />
- </TTDDialogTab>
- </TTDDialogTabs>
- </Dialog>
- );
-};
diff --git a/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx b/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx
index 30add91e59..439f92844d 100644
--- a/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx
+++ b/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx
@@ -7,10 +7,7 @@ import { isMemberOf } from "../../utils";
const TTDDialogTabs = (
props: {
children: ReactNode;
- } & (
- | { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }
- | { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" }
- ),
+ } & { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" },
) => {
const setAppState = useExcalidrawSetAppState();
@@ -39,13 +36,6 @@ const TTDDialogTabs = (
}
}
if (
- props.dialog === "settings" &&
- isMemberOf(["text-to-diagram", "diagram-to-code"], tab)
- ) {
- setAppState({
- openDialog: { name: props.dialog, tab, source: "settings" },
- });
- } else if (
props.dialog === "ttd" &&
isMemberOf(["text-to-diagram", "mermaid"], tab)
) {
diff --git a/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx b/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx
index 0dacaebf98..171c1c71e7 100644
--- a/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx
+++ b/packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx
@@ -13,6 +13,7 @@ const DropdownMenuItemLink = ({
onSelect,
className = "",
selected,
+ rel = "noreferrer",
...rest
}: {
href: string;
@@ -22,6 +23,7 @@ const DropdownMenuItemLink = ({
className?: string;
selected?: boolean;
onSelect?: (event: Event) => void;
+ rel?: string;
} & React.AnchorHTMLAttributes<HTMLAnchorElement>) => {
const handleClick = useHandleDropdownMenuItemClick(rest.onClick, onSelect);
diff --git a/packages/excalidraw/data/magic.ts b/packages/excalidraw/data/magic.ts
deleted file mode 100644
index 883a3bdb90..0000000000
--- a/packages/excalidraw/data/magic.ts
+++ /dev/null
@@ -1,105 +0,0 @@
-import { THEME } from "../constants";
-import type { Theme } from "../element/types";
-import type { DataURL } from "../types";
-import type { OpenAIInput, OpenAIOutput } from "./ai/types";
-
-export type MagicCacheData =
- | {
- status: "pending";
- }
- | { status: "done"; html: string }
- | {
- status: "error";
- message?: string;
- code: "ERR_GENERATION_INTERRUPTED" | string;
- };
-
-const SYSTEM_PROMPT = `You are a skilled front-end developer who builds interactive prototypes from wireframes, and is an expert at CSS Grid and Flex design.
-Your role is to transform low-fidelity wireframes into working front-end HTML code.
-
-YOU MUST FOLLOW FOLLOWING RULES:
-
-- Use HTML, CSS, JavaScript to build a responsive, accessible, polished prototype
-- Leverage Tailwind for styling and layout (import as script <script src="https://cdn.tailwindcss.com"></script>)
-- Inline JavaScript when needed
-- Fetch dependencies from CDNs when needed (using unpkg or skypack)
-- Source images from Unsplash or create applicable placeholders
-- Interpret annotations as intended vs literal UI
-- Fill gaps using your expertise in UX and business logic
-- generate primarily for desktop UI, but make it responsive.
-- Use grid and flexbox wherever applicable.
-- Convert the wireframe in its entirety, don't omit elements if possible.
-
-If the wireframes, diagrams, or text is unclear or unreadable, refer to provided text for clarification.
-
-Your goal is a production-ready prototype that brings the wireframes to life.
-
-Please output JUST THE HTML file containing your best attempt at implementing the provided wireframes.`;
-
-export async function diagramToHTML({
- image,
- apiKey,
- text,
- theme = THEME.LIGHT,
-}: {
- image: DataURL;
- apiKey: string;
- text: string;
- theme?: Theme;
-}) {
- const body: OpenAIInput.ChatCompletionCreateParamsBase = {
- model: "gpt-4-vision-preview",
- // 4096 are max output tokens allowed for `gpt-4-vision-preview` currently
- max_tokens: 4096,
- temperature: 0.1,
- messages: [
- {
- role: "system",
- content: SYSTEM_PROMPT,
- },
- {
- role: "user",
- content: [
- {
- type: "image_url",
- image_url: {
- url: image,
- detail: "high",
- },
- },
- {
- type: "text",
- text: `Above is the reference wireframe. Please make a new website based on these and return just the HTML file. Also, please make it for the ${theme} theme. What follows are the wireframe's text annotations (if any)...`,
- },
- {
- type: "text",
- text,
- },
- ],
- },
- ],
- };
-
- let result:
- | ({ ok: true } & OpenAIOutput.ChatCompletion)
- | ({ ok: false } & OpenAIOutput.APIError);
-
- const resp = await fetch("https://api.openai.com/v1/chat/completions", {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- Authorization: `Bearer ${apiKey}`,
- },
- body: JSON.stringify(body),
- });
-
- if (resp.ok) {
- const json: OpenAIOutput.ChatCompletion = await resp.json();
- result = { ...json, ok: true };
- } else {
- const json: OpenAIOutput.APIError = await resp.json();
- result = { ...json, ok: false };
- }
-
- return result;
-}
diff --git a/packages/excalidraw/element/index.ts b/packages/excalidraw/element/index.ts
index d6bc8f6156..a9b747681f 100644
--- a/packages/excalidraw/element/index.ts
+++ b/packages/excalidraw/element/index.ts
@@ -46,7 +46,7 @@ export {
dragNewElement,
} from "./dragElements";
export { isTextElement, isExcalidrawElement } from "./typeChecks";
-export { redrawTextBoundingBox } from "./textElement";
+export { redrawTextBoundingBox, getTextFromElements } from "./textElement";
export {
getPerfectElementSize,
getLockedLinearCursorAlignSize,
diff --git a/packages/excalidraw/element/textElement.ts b/packages/excalidraw/element/textElement.ts
index 696f809e6b..9fb4766b6e 100644
--- a/packages/excalidraw/element/textElement.ts
+++ b/packages/excalidraw/element/textElement.ts
@@ -886,3 +886,19 @@ export const getMinTextElementWidth = (
) => {
return measureText("", font, lineHeight).width + BOUND_TEXT_PADDING * 2;
};
+
+/** retrieves text from text elements and concatenates to a single string */
+export const getTextFromElements = (
+ elements: readonly ExcalidrawElement[],
+ separator = "\n\n",
+) => {
+ const text = elements
+ .reduce((acc: string[], element) => {
+ if (isTextElement(element)) {
+ acc.push(element.text);
+ }
+ return acc;
+ }, [])
+ .join(separator);
+ return text;
+};
diff --git a/packages/excalidraw/element/types.ts b/packages/excalidraw/element/types.ts
index 33372764b7..4322cf8510 100644
--- a/packages/excalidraw/element/types.ts
+++ b/packages/excalidraw/element/types.ts
@@ -12,7 +12,6 @@ import type {
Merge,
ValueOf,
} from "../utility-types";
-import type { MagicCacheData } from "../data/magic";
export type ChartType = "bar" | "line";
export type FillStyle = "hachure" | "cross-hatch" | "solid" | "zigzag";
@@ -101,11 +100,22 @@ export type ExcalidrawEmbeddableElement = _ExcalidrawElementBase &
type: "embeddable";
}>;
+export type MagicGenerationData =
+ | {
+ status: "pending";
+ }
+ | { status: "done"; html: string }
+ | {
+ status: "error";
+ message?: string;
+ code: "ERR_GENERATION_INTERRUPTED" | string;
+ };
+
export type ExcalidrawIframeElement = _ExcalidrawElementBase &
Readonly<{
type: "iframe";
// TODO move later to AI-specific frame
- customData?: { generationData?: MagicCacheData };
+ customData?: { generationData?: MagicGenerationData };
}>;
export type ExcalidrawIframeLikeElement =
diff --git a/packages/excalidraw/index.tsx b/packages/excalidraw/index.tsx
index ce8bdbe876..2e34eb7a6d 100644
--- a/packages/excalidraw/index.tsx
+++ b/packages/excalidraw/index.tsx
@@ -213,6 +213,7 @@ export {
hashString,
isInvisiblySmallElement,
getNonDeletedElements,
+ getTextFromElements,
} from "./element";
export { defaultLang, useI18n, languages } from "./i18n";
export {
@@ -287,3 +288,6 @@ export {
isElementInsideBBox,
elementPartiallyOverlapsWithOrContainsBBox,
} from "../utils/withinBounds";
+
+export { DiagramToCodePlugin } from "./components/DiagramToCodePlugin/DiagramToCodePlugin";
+export { getDataURL } from "./data/blob";
diff --git a/packages/excalidraw/locales/en.json b/packages/excalidraw/locales/en.json
index b67624efe5..882293c8a8 100644
--- a/packages/excalidraw/locales/en.json
+++ b/packages/excalidraw/locales/en.json
@@ -272,8 +272,7 @@
"laser": "Laser pointer",
"hand": "Hand (panning tool)",
"extraTools": "More tools",
- "mermaidToExcalidraw": "Mermaid to Excalidraw",
- "magicSettings": "AI settings"
+ "mermaidToExcalidraw": "Mermaid to Excalidraw"
},
"element": {
"rectangle": "Rectangle",
diff --git a/packages/excalidraw/types.ts b/packages/excalidraw/types.ts
index 3b107f26e2..0721f932dd 100644
--- a/packages/excalidraw/types.ts
+++ b/packages/excalidraw/types.ts
@@ -325,14 +325,6 @@ export interface AppState {
openDialog:
| null
| { name: "imageExport" | "help" | "jsonExport" }
- | {
- name: "settings";
- source:
- | "tool" // when magicframe tool is selected
- | "generation" // when magicframe generate button is clicked
- | "settings"; // when AI settings dialog is explicitly invoked
- tab: "text-to-diagram" | "diagram-to-code";
- }
| { name: "ttd"; tab: "text-to-diagram" | "mermaid" }
| { name: "commandPalette" };
/**
@@ -655,6 +647,8 @@ export type AppClassProperties = {
dismissLinearEditor: App["dismissLinearEditor"];
flowChartCreator: App["flowChartCreator"];
getEffectiveGridSize: App["getEffectiveGridSize"];
+ setPlugins: App["setPlugins"];
+ plugins: App["plugins"];
};
export type PointerDownState = Readonly<{
@@ -842,3 +836,8 @@ export type PendingExcalidrawElements = ExcalidrawElement[];
export type NullableGridSize =
| (AppState["gridSize"] & MakeBrand<"NullableGridSize">)
| null;
+
+export type GenerateDiagramToCode = (props: {
+ frame: ExcalidrawMagicFrameElement;
+ children: readonly ExcalidrawElement[];
+}) => MaybePromise<{ html: string }>;
diff --git a/packages/excalidraw/utils.ts b/packages/excalidraw/utils.ts
index 3216eba5c2..0f3d11f717 100644
--- a/packages/excalidraw/utils.ts
+++ b/packages/excalidraw/utils.ts
@@ -1166,3 +1166,11 @@ export const promiseTry = async <TValue, TArgs extends unknown[]>(
export const isAnyTrue = (...args: boolean[]): boolean =>
Math.max(...args.map((arg) => (arg ? 1 : 0))) > 0;
+
+export const safelyParseJSON = (json: string): Record<string, any> | null => {
+ try {
+ return JSON.parse(json);
+ } catch {
+ return null;
+ }
+};