Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

feat: new AI command bar (#2445)

Ivan Starkov19f1d5d73431
diff --git a/apps/builder/app/builder/features/ai/ai-command-bar.tsx b/apps/builder/app/builder/features/ai/ai-command-bar.tsx
index 846770f665..c8d4364fbb 100644
--- a/apps/builder/app/builder/features/ai/ai-command-bar.tsx
+++ b/apps/builder/app/builder/features/ai/ai-command-bar.tsx
@@ -15,6 +15,7 @@ import {
Text,
Tooltip,
theme,
+ toast,
useDisableCanvasPointerEvents,
} from "@webstudio-is/design-system";
import {
@@ -34,28 +35,9 @@ import {
import { $isAiCommandBarVisible } from "~/shared/nano-states";
import { useMediaRecorder } from "./hooks/media-recorder";
import { useLongPressToggle } from "./hooks/long-press-toggle";
-import { restAi } from "~/shared/router-utils";
import { AiCommandBarButton } from "./ai-button";
-
-const fetchTranscription = async (file: File) => {
- const formData = new FormData();
- formData.append("file", file);
-
- const response = await fetch(`${restAi()}/audio/transcriptions`, {
- method: "POST",
- body: formData,
- });
-
- if (response.ok === false) {
- // @todo: show error
- return;
- }
-
- // @todo add response parsing
- const { text } = await response.json();
-
- return text;
-};
+import { fetchTranscription } from "./ai-fetch-transcription";
+import { fetchResult } from "./ai-fetch-result";
type PartialButtonProps<T = ComponentPropsWithoutRef<typeof Button>> = {
[key in keyof T]?: T[key];
@@ -63,11 +45,13 @@ type PartialButtonProps<T = ComponentPropsWithoutRef<typeof Button>> = {
export const AiCommandBar = () => {
const [value, setValue] = useState("");
+ const [prompts, setPrompts] = useState<string[]>([]);
const [open, setOpen] = useState(false);
const [isAudioTranscribing, setIsAudioTranscribing] = useState(false);
+ const [isAiRequesting, setIsAiRequesting] = useState(false);
const isAiCommandBarVisible = useStore($isAiCommandBarVisible);
const recordButtonRef = useRef<HTMLButtonElement>(null);
- const uploadIdRef = useRef(0);
+ const guardIdRef = useRef(0);
const { enableCanvasPointerEvents, disableCanvasPointerEvents } =
useDisableCanvasPointerEvents();
@@ -79,18 +63,23 @@ export const AiCommandBar = () => {
} = useMediaRecorder({
onComplete: async (file) => {
setIsAudioTranscribing(true);
- uploadIdRef.current++;
- const uploadId = uploadIdRef.current;
+ guardIdRef.current++;
+ const guardId = guardIdRef.current;
const text = await fetchTranscription(file);
- if (uploadId !== uploadIdRef.current) {
+ if (guardId !== guardIdRef.current) {
return;
}
setValue((previousText) => `${previousText} ${text}`);
setIsAudioTranscribing(false);
+
+ setValue((value) => {
+ Promise.resolve(true).then(() => handleAiRequest(value));
+ return value;
+ });
},
onReportSoundAmplitude: (amplitude) => {
recordButtonRef.current?.style.setProperty(
- "--amplitude",
+ "--ws-ai-command-bar-amplitude",
amplitude.toString()
);
},
@@ -111,6 +100,37 @@ export const AiCommandBar = () => {
},
});
+ const handleAiRequest = async (prompt: string) => {
+ setIsAiRequesting(true);
+ guardIdRef.current++;
+ const guardId = guardIdRef.current;
+
+ // Skip Abort Logic for now
+ try {
+ await fetchResult(prompt, new AbortController().signal);
+ if (guardId !== guardIdRef.current) {
+ return;
+ }
+
+ setPrompts((previousPrompts) => [...previousPrompts, prompt]);
+
+ setValue("");
+ } catch (error) {
+ // eslint-disable-next-line no-console
+ console.error(error);
+ if (error instanceof Error) {
+ toast(error.message);
+ return;
+ }
+ toast("Something went wrong");
+ }
+ setIsAiRequesting(false);
+ };
+
+ const handleAiButtonClick = () => {
+ handleAiRequest(value);
+ };
+
if (isAiCommandBarVisible === false) {
return;
}
@@ -127,7 +147,7 @@ export const AiCommandBar = () => {
let aiButtonTooltip: string | undefined = "Generate AI results";
let aiButtonDisabled = value.length === 0;
- const aiButtonPending = false;
+ let aiButtonPending = false;
if (isAudioTranscribing) {
textAreaPlaceholder = "Transcribing voice...";
@@ -140,7 +160,7 @@ export const AiCommandBar = () => {
recordButtonProps = {
onClick: (event: MouseEvent<HTMLButtonElement>) => {
// Cancel transcription
- uploadIdRef.current++;
+ guardIdRef.current++;
setIsAudioTranscribing(false);
},
};
@@ -164,6 +184,26 @@ export const AiCommandBar = () => {
aiButtonTooltip = undefined;
}
+ if (isAiRequesting) {
+ textAreaDisabled = true;
+
+ recordButtonTooltipContent = "Cancel";
+ recordButtonColor = "neutral";
+ recordButtonProps = {
+ onClick: (event: MouseEvent<HTMLButtonElement>) => {
+ // Cancel AI request
+ guardIdRef.current++;
+ setIsAiRequesting(false);
+ alert("Not fully implemented, should abort changes");
+ },
+ };
+ recordButtonIcon = <LargeXIcon />;
+
+ aiButtonTooltip = undefined;
+ aiButtonDisabled = true;
+ aiButtonPending = true;
+ }
+
return (
<Box
css={{
@@ -180,7 +220,7 @@ export const AiCommandBar = () => {
<CommandBar
open={open}
onOpenChange={setOpen}
- content={<CommandBarContent />}
+ content={<CommandBarContent prompts={prompts} />}
>
<CommandBarTrigger>
<CommandBarButton color="dark-ghost">
@@ -206,7 +246,7 @@ export const AiCommandBar = () => {
onKeyDown={(event) => {
if (event.key === "Enter" && event.shiftKey === false) {
event.preventDefault();
- // @todo add text submit here
+ handleAiRequest(value);
}
}}
/>
@@ -222,8 +262,8 @@ export const AiCommandBar = () => {
<CommandBarButton
ref={recordButtonRef}
css={{
- "--amplitude": 0,
- opacity: "calc(1 - 0.5 * var(--amplitude, 0))",
+ "--ws-ai-command-bar-amplitude": 0,
+ opacity: "calc(1 - 0.5 * var(--ws-ai-command-bar-amplitude, 0))",
transition: "opacity 0.1s ease-in-out",
}}
color={recordButtonColor}
@@ -243,6 +283,7 @@ export const AiCommandBar = () => {
color="gradient"
data-pending={aiButtonPending}
disabled={aiButtonDisabled}
+ onClick={handleAiButtonClick}
>
<AiIcon />
</AiCommandBarButton>
@@ -252,7 +293,7 @@ export const AiCommandBar = () => {
);
};
-const CommandBarContent = () => {
+const CommandBarContent = (props: { prompts: string[] }) => {
const shortcutText = "⌘⇧Q";
return (
<>
@@ -296,19 +337,15 @@ const CommandBarContent = () => {
Previous prompts
</Text>
<div />
- <CommandBarContentPrompt>
- Make a new section with a contact form
- </CommandBarContentPrompt>
- <CommandBarContentPrompt>
- Add an image of a cat smiling wearing eyeglasses
- </CommandBarContentPrompt>
- <CommandBarContentPrompt>
- Write a breakup letter for my girlfriend and make it long enough to
- break into multiple lines
- </CommandBarContentPrompt>
- <CommandBarContentPrompt>
- Make all headings bold
- </CommandBarContentPrompt>
+ <ScrollArea css={{ maxHeight: theme.spacing[29] }}>
+ <Grid gap={2}>
+ {props.prompts.map((prompt, index) => (
+ <CommandBarContentPrompt key={index}>
+ {prompt}
+ </CommandBarContentPrompt>
+ ))}
+ </Grid>
+ </ScrollArea>
</CommandBarContentSection>
</>
);
diff --git a/apps/builder/app/builder/features/ai/ai-fetch-result.ts b/apps/builder/app/builder/features/ai/ai-fetch-result.ts
new file mode 100644
index 0000000000..8a3473ba6e
--- /dev/null
+++ b/apps/builder/app/builder/features/ai/ai-fetch-result.ts
@@ -0,0 +1,289 @@
+// import { restAi } from "~/shared/router-utils";
+
+import { z } from "zod";
+import {
+ copywriter,
+ type operations,
+ handleAiRequest,
+ commandDetect,
+} from "@webstudio-is/ai";
+import { createCssEngine } from "@webstudio-is/css-engine";
+import {
+ generateJsxElement,
+ generateJsxChildren,
+ getIndexesWithinAncestors,
+ getStyleRules,
+ idAttribute,
+ componentAttribute,
+} from "@webstudio-is/react-sdk";
+import { createScope, findTreeInstanceIds } from "@webstudio-is/sdk";
+import { computed } from "nanostores";
+import { getMapValuesByKeysSet } from "~/shared/array-utils";
+import {
+ breakpointsStore,
+ dataSourcesStore,
+ instancesStore,
+ projectStore,
+ propsStore,
+ registeredComponentMetasStore,
+ selectedInstanceSelectorStore,
+ styleSourceSelectionsStore,
+ stylesStore,
+} from "~/shared/nano-states";
+import { applyOperations, patchTextInstance } from "./apply-operations";
+import { restAi } from "~/shared/router-utils";
+import { RequestParamsSchema } from "~/routes/rest.ai";
+import untruncateJson from "untruncate-json";
+import { traverseTemplate } from "@webstudio-is/jsx-utils";
+
+export const fetchResult = async (
+ prompt: string,
+ abortSignal: AbortSignal
+): Promise<void> => {
+ const project = projectStore.get();
+ const selectedInstanceSelector = selectedInstanceSelectorStore.get();
+ const availableComponentsNames = $availableComponentsNames.get();
+ const jsx = $jsx.get();
+
+ const requestParams = {
+ jsx: jsx ? JSON.stringify(jsx) : "",
+ components: availableComponentsNames,
+ projectId: project?.id,
+ instanceId: selectedInstanceSelector?.[0] ?? "",
+ prompt,
+ };
+
+ if (
+ RequestParamsSchema.omit({ command: true }).safeParse(requestParams)
+ .success === false
+ ) {
+ throw new Error("Invalid prompt data");
+ }
+
+ const commandsResponse = await handleAiRequest<commandDetect.Response>(
+ fetch(restAi("detect"), {
+ method: "POST",
+ body: JSON.stringify({ prompt: requestParams.prompt }),
+ signal: abortSignal,
+ }),
+ { signal: abortSignal }
+ );
+
+ if (
+ commandsResponse.success === false ||
+ commandsResponse.type === "stream"
+ ) {
+ // Commands detection is not using streaming
+ // therefore this should never be the case.
+ // The check is here just for type safety.
+ if (commandsResponse.type === "stream") {
+ // eslint-disable-next-line no-console
+ throw new Error("Something went wrong.");
+ }
+
+ if (abortSignal.aborted === false) {
+ throw new Error(commandsResponse.data.message);
+ }
+
+ throw new Error("Unknown error");
+ }
+
+ const promises = await Promise.allSettled(
+ commandsResponse.data.map((command) =>
+ handleAiRequest<operations.Response>(
+ fetch(restAi(), {
+ method: "POST",
+ body: JSON.stringify({ ...requestParams, command }),
+ signal: abortSignal,
+ }),
+ {
+ signal: abortSignal,
+ onChunk: (operationId, { completion }) => {
+ if (operationId === "copywriter") {
+ try {
+ const jsonResponse = z
+ .array(copywriter.TextInstanceSchema)
+ .parse(JSON.parse(untruncateJson(completion)));
+
+ const currenTextInstance = jsonResponse.pop();
+
+ if (currenTextInstance === undefined) {
+ return;
+ }
+
+ patchTextInstance(currenTextInstance);
+ } catch {
+ /**/
+ }
+ }
+ },
+ }
+ )
+ )
+ );
+
+ const errors = [];
+
+ for (const promise of promises) {
+ if (promise.status === "fulfilled") {
+ const result = promise.value;
+
+ if (result.success === false) {
+ errors.push(result.data.message);
+ continue;
+ }
+
+ if (result.type !== "json") {
+ continue;
+ }
+
+ if (result.id === "operations") {
+ restoreComponentsNamespace(result.data);
+ applyOperations(result.data);
+ continue;
+ }
+
+ // Handle other commands responses below.
+ // ...
+ //
+ } else if (promise.status === "rejected") {
+ errors.push(promise.reason);
+ }
+ }
+
+ if (errors.length > 0) {
+ throw new Error(errors.join("\n"));
+ }
+};
+
+const $availableComponentsNames = computed(
+ [registeredComponentMetasStore],
+ (metas) => {
+ const exclude = ["Body", "Slot"];
+
+ return [...metas.keys()]
+ .filter((name) => !exclude.includes(name))
+ .map(parseComponentName);
+ }
+);
+
+// The LLM gets a list of available component names
+// therefore we need to replace the component namespace with a LLM-friendly one
+// preserving context eg. Radix.Dialog instead of just Dialog
+const parseComponentName = (name: string) =>
+ name.replace("@webstudio-is/sdk-components-react-radix:", "Radix.");
+// When AI generation is done we need to restore components namespaces.
+const restoreComponentsNamespace = (operations: operations.WsOperations) => {
+ for (const operation of operations) {
+ if (operation.operation === "insertTemplate") {
+ traverseTemplate(operation.template, (node) => {
+ if (node.type === "instance" && node.component.startsWith("Radix.")) {
+ node.component =
+ "@webstudio-is/sdk-components-react-radix:" +
+ node.component.slice("Radix.".length);
+ }
+ });
+ }
+ }
+};
+
+const $jsx = computed(
+ [
+ selectedInstanceSelectorStore,
+ instancesStore,
+ propsStore,
+ dataSourcesStore,
+ registeredComponentMetasStore,
+ breakpointsStore,
+ stylesStore,
+ styleSourceSelectionsStore,
+ ],
+ (
+ selectedInstanceSelector,
+ instances,
+ props,
+ dataSources,
+ metas,
+ breakpoints,
+ styles,
+ styleSourceSelections
+ ) => {
+ if (selectedInstanceSelector === undefined) {
+ return;
+ }
+
+ const [rootInstanceId] = selectedInstanceSelector;
+ const instance = instances.get(rootInstanceId);
+ if (instance === undefined) {
+ return;
+ }
+ const indexesWithinAncestors = getIndexesWithinAncestors(metas, instances, [
+ rootInstanceId,
+ ]);
+ const scope = createScope();
+
+ const jsx = generateJsxElement({
+ scope,
+ instance,
+ props,
+ dataSources,
+ indexesWithinAncestors,
+ children: generateJsxChildren({
+ scope,
+ children: instance.children,
+ instances,
+ props,
+ dataSources,
+ indexesWithinAncestors,
+ }),
+ });
+
+ const treeInstanceIds = findTreeInstanceIds(instances, rootInstanceId);
+
+ const treeStyleSourceSelections = new Map(
+ getMapValuesByKeysSet(styleSourceSelections, treeInstanceIds).map(
+ (styleSourceSelection) => [
+ styleSourceSelection.instanceId,
+ styleSourceSelection,
+ ]
+ )
+ );
+
+ const engine = createCssEngine({ name: "ssr" });
+
+ const styleRules = getStyleRules(styles, treeStyleSourceSelections);
+ for (const { breakpointId, instanceId, state, style } of styleRules) {
+ engine.addStyleRule(`[${idAttribute}="${instanceId}"]${state ?? ""}`, {
+ breakpoint: breakpointId,
+ style,
+ });
+ }
+
+ return `<style>{\`${engine.cssText.replace(/\n/gm, " ")}\`}</style>${jsx
+ .replace(new RegExp(`${componentAttribute}="[^"]+"`, "g"), "")
+ .replace(/\n(data-)/g, " $1")}`;
+ }
+);
+
+/*
+export const fetchResult = async (text: string) => {
+ const formData = new FormData();
+ formData.append("file", file);
+
+ const response = await fetch(`${restAi()}/blabla`, {
+ method: "POST",
+ body: formData,
+ });
+
+ if (response.ok === false) {
+ // @todo: show error
+ return;
+ }
+
+ // @todo add response parsing
+ const { text } = await response.json();
+
+ // return text;
+ await new Promise((resolve) => setTimeout(resolve, 10000));
+};
+*/
diff --git a/apps/builder/app/builder/features/ai/ai-fetch-transcription.ts b/apps/builder/app/builder/features/ai/ai-fetch-transcription.ts
new file mode 100644
index 0000000000..7a81f32b7c
--- /dev/null
+++ b/apps/builder/app/builder/features/ai/ai-fetch-transcription.ts
@@ -0,0 +1,21 @@
+import { restAi } from "~/shared/router-utils";
+
+export const fetchTranscription = async (file: File) => {
+ const formData = new FormData();
+ formData.append("file", file);
+
+ const response = await fetch(`${restAi()}/audio/transcriptions`, {
+ method: "POST",
+ body: formData,
+ });
+
+ if (response.ok === false) {
+ // @todo: show error
+ return;
+ }
+
+ // @todo add response parsing
+ const { text } = await response.json();
+
+ return text;
+};