Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

feat: Audio transcribe (#2438)

Ivan Starkov9ff675a4359d
diff --git a/apps/builder/app/builder/features/ai/ai-command-bar.tsx b/apps/builder/app/builder/features/ai/ai-command-bar.tsx
index 9ec7f9a012..d5c12e1b93 100644
--- a/apps/builder/app/builder/features/ai/ai-command-bar.tsx
+++ b/apps/builder/app/builder/features/ai/ai-command-bar.tsx
@@ -20,18 +20,87 @@ import {
MicIcon,
ChevronUpIcon,
ExternalLinkIcon,
+ StopIcon,
} from "@webstudio-is/icons";
-import { useState } from "react";
+import { useRef, useState } from "react";
import { $isAiCommandBarVisible } from "~/shared/nano-states";
+import { useMediaRecorder } from "./hooks/media-recorder";
+import { useLongPressToggle } from "./hooks/long-press-toggle";
+import { restAi } from "~/shared/router-utils";
+
+const fetchTranscription = async (file: File) => {
+ const formData = new FormData();
+ formData.append("file", file);
+
+ const response = await fetch(`${restAi()}/audio/transcriptions`, {
+ method: "POST",
+ body: formData,
+ });
+
+ if (response.ok === false) {
+ // @todo: show error
+ return;
+ }
+
+ // @todo add response parsing
+ const { text } = await response.json();
+
+ return text;
+};
export const AiCommandBar = () => {
+ const [value, setValue] = useState("");
const [open, setOpen] = useState(false);
+ const [isAudioTranscribing, setIsAudioTranscribing] = useState(false);
const isAiCommandBarVisible = useStore($isAiCommandBarVisible);
+ const recordButtonRef = useRef<HTMLButtonElement>(null);
+ const uploadIdRef = useRef(0);
+
+ const {
+ start,
+ stop,
+ cancel,
+ state: mediaRecorderState,
+ } = useMediaRecorder({
+ onComplete: async (file) => {
+ setIsAudioTranscribing(true);
+ uploadIdRef.current++;
+ const uploadId = uploadIdRef.current;
+ const text = await fetchTranscription(file);
+ if (uploadId !== uploadIdRef.current) {
+ return;
+ }
+ setValue(text);
+ setIsAudioTranscribing(false);
+ },
+ onReportSoundAmplitude: (amplitude) => {
+ recordButtonRef.current?.style.setProperty(
+ "--amplitude",
+ amplitude.toString()
+ );
+ },
+ });
+
+ const longPressToggleProps = useLongPressToggle({
+ onStart: () => {
+ setValue("");
+ start();
+ },
+ onEnd: stop,
+ onCancel: cancel,
+ });
if (isAiCommandBarVisible === false) {
return;
}
+ const textAreaDisabled =
+ mediaRecorderState === "recording" || isAudioTranscribing;
+
+ const recordButtonDisabled = isAudioTranscribing;
+ const aiButtonDisabled =
+ mediaRecorderState === "recording" || isAudioTranscribing;
+
return (
<Box
css={{
@@ -65,14 +134,32 @@ export const AiCommandBar = () => {
}}
>
<ScrollArea css={{ maxHeight: theme.spacing[29] }}>
- <AutogrowTextArea autoFocus placeholder="Enter value..." />
+ <AutogrowTextArea
+ autoFocus
+ disabled={textAreaDisabled}
+ placeholder="Enter value..."
+ value={value}
+ onChange={setValue}
+ />
</ScrollArea>
</Grid>
- <CommandBarButton color="dark-ghost">
- <MicIcon />
+ <CommandBarButton
+ disabled={recordButtonDisabled}
+ ref={recordButtonRef}
+ css={{
+ opacity: "calc(1 - 0.5 * var(--amplitude, 0))",
+ transition: "opacity 0.1s ease-in-out",
+ }}
+ color={
+ mediaRecorderState === "recording" ? "destructive" : "dark-ghost"
+ }
+ {...longPressToggleProps}
+ >
+ {mediaRecorderState === "recording" ? <StopIcon /> : <MicIcon />}
</CommandBarButton>
- <CommandBarButton color="gradient">
+
+ <CommandBarButton color="gradient" disabled={aiButtonDisabled}>
<AiIcon />
</CommandBarButton>
</CommandBar>
diff --git a/apps/builder/app/builder/features/ai/hooks/media-recorder.ts b/apps/builder/app/builder/features/ai/hooks/media-recorder.ts
index 8d3dc30173..01adf5bf44 100644
--- a/apps/builder/app/builder/features/ai/hooks/media-recorder.ts
+++ b/apps/builder/app/builder/features/ai/hooks/media-recorder.ts
@@ -7,10 +7,7 @@ const DEFAULT_OPTIONS: MediaRecorderOptions = {
};
export const useMediaRecorder = (
- {
- onComplete,
- onReportSoundAmplitude,
- }: {
+ props: {
onComplete: (file: File) => void;
onReportSoundAmplitude?: (amplitude: number) => void;
},
@@ -24,6 +21,9 @@ export const useMediaRecorder = (
const [error, setError] = useState<Error>();
const [state, setState] = useState<"inactive" | "recording">("inactive");
+ const onComplete = useEffectEvent(props.onComplete);
+ const onReportSoundAmplitude = useEffectEvent(props.onReportSoundAmplitude);
+
const start = useEffectEvent(async () => {
isActiveRef.current = true;
const chunks: Blob[] = [];
diff --git a/apps/builder/app/routes/rest.ai.audio.transcriptions.ts b/apps/builder/app/routes/rest.ai.audio.transcriptions.ts
new file mode 100644
index 0000000000..a1b181ae1b
--- /dev/null
+++ b/apps/builder/app/routes/rest.ai.audio.transcriptions.ts
@@ -0,0 +1,41 @@
+import type { ActionArgs } from "@remix-run/node";
+
+// @todo: move to AI package
+export const action = async ({ request }: ActionArgs) => {
+ // @todo: validate request
+ const formData = await request.formData();
+ formData.append("model", "whisper-1");
+
+ const response = await fetch(
+ "https://api.openai.com/v1/audio/transcriptions",
+ {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${process.env.OPENAI_KEY}`,
+ },
+ body: formData,
+ }
+ );
+
+ if (response.ok === false) {
+ const txt = await response.text();
+
+ // eslint-disable-next-line no-console
+ console.error("ERROR", response.status, txt);
+
+ return {
+ success: false,
+ error: {
+ status: response.status,
+ message: txt,
+ },
+ };
+ }
+
+ const data = await response.json();
+
+ return {
+ success: true,
+ text: data.text,
+ };
+};
diff --git a/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsx b/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsx
index baee1a5e36..fc62a13d23 100644
--- a/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsx
+++ b/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsx
@@ -33,7 +33,8 @@ const style = css(commonStyle, {
},
"&:disabled": {
color: theme.colors.foregroundDisabled,
- background: theme.colors.backgroundInputDisabled,
+ // @todo: Ask Taylor if we should use the same color as the placeholder
+ opacity: 0.6,
},
variants: {
state: {