Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
feat: Audio transcribe (#2438)
Ivan Starkov9ff675a4359d
diff --git a/apps/builder/app/builder/features/ai/ai-command-bar.tsx b/apps/builder/app/builder/features/ai/ai-command-bar.tsxindex 9ec7f9a012..d5c12e1b93 100644--- a/apps/builder/app/builder/features/ai/ai-command-bar.tsx+++ b/apps/builder/app/builder/features/ai/ai-command-bar.tsx@@ -20,18 +20,87 @@ import {MicIcon,ChevronUpIcon,ExternalLinkIcon,+ StopIcon,} from "@webstudio-is/icons";-import { useState } from "react";+import { useRef, useState } from "react";import { $isAiCommandBarVisible } from "~/shared/nano-states";+import { useMediaRecorder } from "./hooks/media-recorder";+import { useLongPressToggle } from "./hooks/long-press-toggle";+import { restAi } from "~/shared/router-utils";++const fetchTranscription = async (file: File) => {+ const formData = new FormData();+ formData.append("file", file);++ const response = await fetch(`${restAi()}/audio/transcriptions`, {+ method: "POST",+ body: formData,+ });++ if (response.ok === false) {+ // @todo: show error+ return;+ }++ // @todo add response parsing+ const { text } = await response.json();++ return text;+};export const AiCommandBar = () => {+ const [value, setValue] = useState("");const [open, setOpen] = useState(false);+ const [isAudioTranscribing, setIsAudioTranscribing] = useState(false);const isAiCommandBarVisible = useStore($isAiCommandBarVisible);+ const recordButtonRef = useRef<HTMLButtonElement>(null);+ const uploadIdRef = useRef(0);++ const {+ start,+ stop,+ cancel,+ state: mediaRecorderState,+ } = useMediaRecorder({+ onComplete: async (file) => {+ setIsAudioTranscribing(true);+ uploadIdRef.current++;+ const uploadId = uploadIdRef.current;+ const text = await fetchTranscription(file);+ if (uploadId !== uploadIdRef.current) {+ return;+ }+ setValue(text);+ setIsAudioTranscribing(false);+ },+ onReportSoundAmplitude: (amplitude) => {+ recordButtonRef.current?.style.setProperty(+ "--amplitude",+ amplitude.toString()+ );+ },+ });++ const longPressToggleProps = useLongPressToggle({+ onStart: () => {+ setValue("");+ start();+ },+ onEnd: stop,+ onCancel: cancel,+ });if (isAiCommandBarVisible === false) {return;}+ const textAreaDisabled =+ mediaRecorderState === "recording" || isAudioTranscribing;++ const recordButtonDisabled = isAudioTranscribing;+ const aiButtonDisabled =+ mediaRecorderState === "recording" || isAudioTranscribing;+return (<Boxcss={{@@ -65,14 +134,32 @@ export const AiCommandBar = () => {}}><ScrollArea css={{ maxHeight: theme.spacing[29] }}>- <AutogrowTextArea autoFocus placeholder="Enter value..." />+ <AutogrowTextArea+ autoFocus+ disabled={textAreaDisabled}+ placeholder="Enter value..."+ value={value}+ onChange={setValue}+ /></ScrollArea></Grid>- <CommandBarButton color="dark-ghost">- <MicIcon />+ <CommandBarButton+ disabled={recordButtonDisabled}+ ref={recordButtonRef}+ css={{+ opacity: "calc(1 - 0.5 * var(--amplitude, 0))",+ transition: "opacity 0.1s ease-in-out",+ }}+ color={+ mediaRecorderState === "recording" ? "destructive" : "dark-ghost"+ }+ {...longPressToggleProps}+ >+ {mediaRecorderState === "recording" ? <StopIcon /> : <MicIcon />}</CommandBarButton>- <CommandBarButton color="gradient">++ <CommandBarButton color="gradient" disabled={aiButtonDisabled}><AiIcon /></CommandBarButton></CommandBar>diff --git a/apps/builder/app/builder/features/ai/hooks/media-recorder.ts b/apps/builder/app/builder/features/ai/hooks/media-recorder.tsindex 8d3dc30173..01adf5bf44 100644--- a/apps/builder/app/builder/features/ai/hooks/media-recorder.ts+++ b/apps/builder/app/builder/features/ai/hooks/media-recorder.ts@@ -7,10 +7,7 @@ const DEFAULT_OPTIONS: MediaRecorderOptions = {};export const useMediaRecorder = (- {- onComplete,- onReportSoundAmplitude,- }: {+ props: {onComplete: (file: File) => void;onReportSoundAmplitude?: (amplitude: number) => void;},@@ -24,6 +21,9 @@ export const useMediaRecorder = (const [error, setError] = useState<Error>();const [state, setState] = useState<"inactive" | "recording">("inactive");+ const onComplete = useEffectEvent(props.onComplete);+ const onReportSoundAmplitude = useEffectEvent(props.onReportSoundAmplitude);+const start = useEffectEvent(async () => {isActiveRef.current = true;const chunks: Blob[] = [];diff --git a/apps/builder/app/routes/rest.ai.audio.transcriptions.ts b/apps/builder/app/routes/rest.ai.audio.transcriptions.tsnew file mode 100644index 0000000000..a1b181ae1b--- /dev/null+++ b/apps/builder/app/routes/rest.ai.audio.transcriptions.ts@@ -0,0 +1,41 @@+import type { ActionArgs } from "@remix-run/node";++// @todo: move to AI package+export const action = async ({ request }: ActionArgs) => {+ // @todo: validate request+ const formData = await request.formData();+ formData.append("model", "whisper-1");++ const response = await fetch(+ "https://api.openai.com/v1/audio/transcriptions",+ {+ method: "POST",+ headers: {+ Authorization: `Bearer ${process.env.OPENAI_KEY}`,+ },+ body: formData,+ }+ );++ if (response.ok === false) {+ const txt = await response.text();++ // eslint-disable-next-line no-console+ console.error("ERROR", response.status, txt);++ return {+ success: false,+ error: {+ status: response.status,+ message: txt,+ },+ };+ }++ const data = await response.json();++ return {+ success: true,+ text: data.text,+ };+};diff --git a/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsx b/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsxindex baee1a5e36..fc62a13d23 100644--- a/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsx+++ b/packages/design-system/src/components/ai-command-bar/autogrow-text-area.tsx@@ -33,7 +33,8 @@ const style = css(commonStyle, {},"&:disabled": {color: theme.colors.foregroundDisabled,- background: theme.colors.backgroundInputDisabled,+ // @todo: Ask Taylor if we should use the same color as the placeholder+ opacity: 0.6,},variants: {state: {