Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
feat: Add Media Recorder (#2431)
Ivan Starkov2f4c0643d056
diff --git a/apps/builder/app/builder/features/ai/hooks/effect-event.ts b/apps/builder/app/builder/features/ai/hooks/effect-event.tsnew file mode 100644index 0000000000..223d2cc0b7--- /dev/null+++ b/apps/builder/app/builder/features/ai/hooks/effect-event.ts@@ -0,0 +1,23 @@+import { useRef, useInsertionEffect, useCallback } from "react";++const useSafeInsertionEffect =+ typeof window !== "undefined"+ ? useInsertionEffect+ : (fn: () => void) => {+ fn();+ };++// eslint-disable-next-line @typescript-eslint/ban-types+export const useEffectEvent = <T extends Function>(callback?: T) => {+ const ref = useRef(callback);++ useSafeInsertionEffect(() => {+ ref.current = callback;+ });++ // eslint-disable-next-line react-hooks/exhaustive-deps+ return useCallback<T>(+ ((...args: unknown[]) => ref.current?.(...args)) as never,+ []+ );+};diff --git a/apps/builder/app/builder/features/ai/hooks/long-press-toggle.ts b/apps/builder/app/builder/features/ai/hooks/long-press-toggle.tsnew file mode 100644index 0000000000..dd771d7757--- /dev/null+++ b/apps/builder/app/builder/features/ai/hooks/long-press-toggle.ts@@ -0,0 +1,94 @@+import { type SyntheticEvent, type KeyboardEvent, useRef } from "react";+import { useEffectEvent } from "./effect-event";++type UseClickAndHoldProps = {+ onStart: () => void;+ onEnd: () => void;+ onCancel?: () => void;+ longPressDuration?: number;+};++/**+ * Toggle hook with long-press handling.+ * - `onStart`: First click.+ * - `onEnd`: Second click or long-press release.+ * - `onCancel`: Pointer up outside target during long press.+ */+export const useLongPressToggle = (props: UseClickAndHoldProps) => {+ const currentTarget = useRef<Element>();+ const pointerDownTimeRef = useRef(0);+ const stateRef = useRef<"idle" | "active">("idle");+ const keyMapRef = useRef(new Set<string>());++ const { longPressDuration = 1000 } = props;++ // Wrap to be sure that latest callback is used in event handlers.+ const onStart = useEffectEvent(props.onStart);+ const onEnd = useEffectEvent(props.onEnd);+ const onCancel = useEffectEvent(props.onCancel);++ const handlePointerUp = useEffectEvent((event: Event) => {+ if (stateRef.current === "idle") {+ return;+ }+ const { target } = event;++ if (!(target instanceof Element)) {+ return;+ }++ const time = Date.now() - pointerDownTimeRef.current;+ const isLongPress = time >= longPressDuration;++ if (isLongPress === false) {+ return;+ }++ if (currentTarget.current?.contains(target)) {+ const time = Date.now() - pointerDownTimeRef.current;+ if (time >= longPressDuration) {+ onEnd();+ stateRef.current = "idle";+ }+ return;+ }++ onCancel?.();+ stateRef.current = "idle";+ });++ const onPointerDown = useEffectEvent((event: SyntheticEvent) => {+ if (stateRef.current === "active") {+ onEnd();+ stateRef.current = "idle";+ return;+ }++ stateRef.current = "active";+ currentTarget.current = event.currentTarget;+ pointerDownTimeRef.current = Date.now();+ document.addEventListener("pointerup", handlePointerUp, { once: true });+ onStart();+ });++ const onKeyDown = useEffectEvent((event: KeyboardEvent) => {+ if (keyMapRef.current.has(event.code)) {+ return;+ }++ if (event.code === "Enter" || event.code === "Space") {+ keyMapRef.current.add(event.code);+ onPointerDown(event);+ }+ });++ const onKeyUp = useEffectEvent((event: KeyboardEvent) => {+ if (event.code === "Enter" || event.code === "Space") {+ keyMapRef.current.delete(event.code);++ handlePointerUp(event.nativeEvent);+ }+ });++ return { onPointerDown, onKeyDown, onKeyUp };+};diff --git a/apps/builder/app/builder/features/ai/hooks/media-recorder.stories.tsx b/apps/builder/app/builder/features/ai/hooks/media-recorder.stories.tsxnew file mode 100644index 0000000000..3280ed6f90--- /dev/null+++ b/apps/builder/app/builder/features/ai/hooks/media-recorder.stories.tsx@@ -0,0 +1,79 @@+import { useRef, useState } from "react";+import { useLongPressToggle } from "./long-press-toggle";+import { useMediaRecorder } from "./media-recorder";+import { Button, Flex, Grid, css } from "@webstudio-is/design-system";++export default {+ title: "Library/Media Recorder",+ argTypes: {+ audioBitsPerSecond: {+ options: [4000, 8000, 16000, 32000, 64000, 128000, 256000],+ control: { type: "radio" },+ },+ },+ args: {+ audioBitsPerSecond: 16000,+ },+};++const playAudio = (file: File) => {+ const blobUrl = URL.createObjectURL(file);+ const audioElement = new Audio(blobUrl);++ audioElement.addEventListener("ended", () => {+ URL.revokeObjectURL(blobUrl);+ });++ audioElement.play();+};++export const MediaRecorder = (options: { audioBitsPerSecond: number }) => {+ const soundRef = useRef<HTMLDivElement>(null);++ const [file, setFile] = useState<File>();++ const { start, stop, cancel, state } = useMediaRecorder(+ {+ onComplete: (file) => {+ setFile(file);+ },+ onReportSoundAmplitude: (amplitude) => {+ soundRef.current?.style.setProperty("--volume", amplitude.toString());+ },+ },+ options+ );++ const clickAndHoldProps = useLongPressToggle({+ onStart: start,+ onEnd: stop,+ onCancel: cancel,+ });++ return (+ <Grid css={{ m: 4 }} gap={2}>+ <Flex ref={soundRef} justify="center" align="center">+ <div className={soundCss()}></div>+ </Flex>+ <Button {...clickAndHoldProps}>+ {state === "inactive" ? "Start recording" : "Recording"}+ </Button>+ {file && (+ <Button color="positive" onClick={() => playAudio(file)}>+ Play, recorded size {file.size}+ </Button>+ )}+ </Grid>+ );+};+MediaRecorder.storyName = "Media Recorder";++const soundCss = css({+ transition: "transform 0.15s ease-in-out",+ borderRadius: "9999px",+ backgroundColor: "#F56565",+ width: "3rem",+ height: "3rem",+ transformOrigin: "center",+ transform: "scale(calc(1 + var(--volume, 0)))",+});diff --git a/apps/builder/app/builder/features/ai/hooks/media-recorder.ts b/apps/builder/app/builder/features/ai/hooks/media-recorder.tsnew file mode 100644index 0000000000..8d3dc30173--- /dev/null+++ b/apps/builder/app/builder/features/ai/hooks/media-recorder.ts@@ -0,0 +1,165 @@+import { useRef, useState } from "react";+import { useEffectEvent } from "./effect-event";++// https://github.com/openai/whisper/discussions/870+const DEFAULT_OPTIONS: MediaRecorderOptions = {+ audioBitsPerSecond: 16000,+};++export const useMediaRecorder = (+ {+ onComplete,+ onReportSoundAmplitude,+ }: {+ onComplete: (file: File) => void;+ onReportSoundAmplitude?: (amplitude: number) => void;+ },+ options = DEFAULT_OPTIONS+) => {+ const disposeRef = useRef<() => void>();++ const cancelRef = useRef(false);+ const isActiveRef = useRef<boolean>(false);+ const idRef = useRef(0);+ const [error, setError] = useState<Error>();+ const [state, setState] = useState<"inactive" | "recording">("inactive");++ const start = useEffectEvent(async () => {+ isActiveRef.current = true;+ const chunks: Blob[] = [];+ idRef.current++;+ const id = idRef.current;++ cancelRef.current = false;++ let stream: MediaStream;++ try {+ stream = await navigator.mediaDevices.getUserMedia({+ audio: true,+ video: false,+ });+ } catch (error) {+ // Not allowed, do not start new recording+ isActiveRef.current = false;++ if (error instanceof Error) {+ setError(error);+ return;+ }+ setError(new Error("Unknown error"));+ return;+ }++ // Recording was stopped, do not start new recording+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment+ // @ts-ignore wrong ts error based on isActiveRef.current = true above it think that isActiveRef.current is always true+ if (isActiveRef.current === false) {+ stream.getAudioTracks().forEach((track) => track.stop());+ return;+ }++ // New recording started, do cleanup and return+ if (id !== idRef.current) {+ stream.getAudioTracks().forEach((track) => track.stop());+ return;+ }++ setError(undefined);+ setState("recording");++ const subtype = MediaRecorder.isTypeSupported("audio/webm; codecs=opus")+ ? "webm"+ : "mp4";+ const mimeType =+ subtype === "webm" ? `audio/${subtype}; codecs=opus` : `audio/${subtype}`;++ const recorder = new MediaRecorder(stream, {+ mimeType,+ ...options,+ });++ const audioContext = new AudioContext();+ const source = audioContext.createMediaStreamSource(stream);+ const analyser = audioContext.createAnalyser();+ source.connect(analyser);++ analyser.fftSize = 2048;+ const bufferLength = analyser.fftSize;+ const dataArray = new Float32Array(bufferLength);++ disposeRef.current = () => {+ source.disconnect();+ analyser.disconnect();+ audioContext.close();+ // Safari bug: Calling `stop` on tracks delays next `getUserMedia` by 3-5s.+ // Chrome: `stop` needed to remove recording tab indicator.+ // @todo: Probably don't stop tracks in Safari, as subsequent getUserMedia blocks the main thread, and cause long-press logic to fail+ stream.getAudioTracks().forEach((track) => track.stop());+ recorder.stop();+ };++ const latestSamples = Array.from({ length: 10 }, () => 1);+ let latestSamplesIndex = 0;++ recorder.ondataavailable = (event) => {+ analyser.getFloatTimeDomainData(dataArray);+ const sampleMaxAmplitude = Math.max(...dataArray);+ latestSamples[latestSamplesIndex] = sampleMaxAmplitude;+ latestSamplesIndex = (latestSamplesIndex + 1) % latestSamples.length;++ // To not normalize amplitude around near zero values+ const normalizeThreshold = 0.1;++ // Normalize amplitude to be between 0 and 1, and against lastest samples.+ // The idea to use latest samples for normalization+ onReportSoundAmplitude?.(+ sampleMaxAmplitude /+ Math.max(normalizeThreshold, Math.max(...latestSamples))+ );++ // New recording started, do cleanup and return+ if (id !== idRef.current) {+ chunks.length = 0;+ return;+ }++ // Cancelled, do cleanup and return+ if (cancelRef.current) {+ chunks.length = 0;+ return;+ }++ chunks.push(event.data);++ if (recorder.state === "inactive") {+ const audioFile = new File(chunks, "recording." + subtype, {+ // Add type to be able to play in audio element+ type: "audio/" + subtype,+ });++ if (audioFile.size > 0) {+ onComplete(audioFile);+ onReportSoundAmplitude?.(0);+ }+ chunks.length = 0;+ }+ };++ recorder.start(50);+ });++ const stop = useEffectEvent(() => {+ isActiveRef.current = false;+ disposeRef.current?.();+ disposeRef.current = undefined;+ setState("inactive");+ });++ const cancel = useEffectEvent(() => {+ cancelRef.current = true;+ stop();+ });++ return { start, stop, cancel, error, state };+};