Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10700): enable browser mic permission prompt for voice input (#6601)
Cindy Lic0d3d0d3ad2c
diff --git a/skyvern-frontend/src/components/SpeechInputButton.tsx b/skyvern-frontend/src/components/SpeechInputButton.tsxindex 6a6c6125a..e814721ec 100644--- a/skyvern-frontend/src/components/SpeechInputButton.tsx+++ b/skyvern-frontend/src/components/SpeechInputButton.tsx@@ -1,3 +1,4 @@+import { MicIcon } from "@/components/icons/MicIcon";import {Tooltip,TooltipContent,@@ -6,24 +7,6 @@ import {} from "@/components/ui/tooltip";import { cn } from "@/util/utils";-function MicIcon({ className }: { className?: string }) {- return (- <svg- xmlns="http://www.w3.org/2000/svg"- viewBox="0 0 15 15"- fill="currentColor"- className={className}- aria-hidden="true"- >- <path- fillRule="evenodd"- clipRule="evenodd"- d="M7.5 1.25a2.25 2.25 0 0 0-2.25 2.25v3.5a2.25 2.25 0 0 0 4.5 0V3.5A2.25 2.25 0 0 0 7.5 1.25Zm-3.75 5.25a3.75 3.75 0 0 0 7.5 0H10a4.75 4.75 0 0 1-9.5 0H3.75ZM7.5 10.25a.75.75 0 0 1 .75.75v1.5h1.25a.75.75 0 0 1 0 1.5H5.5a.75.75 0 0 1 0-1.5h1.25v-1.5A.75.75 0 0 1 7.5 10.25Z"- />- </svg>- );-}-type SpeechInputButtonProps = {isSupported: boolean;isListening: boolean;diff --git a/skyvern-frontend/src/components/icons/MicIcon.tsx b/skyvern-frontend/src/components/icons/MicIcon.tsxnew file mode 100644index 000000000..77e9b3e9f--- /dev/null+++ b/skyvern-frontend/src/components/icons/MicIcon.tsx@@ -0,0 +1,42 @@+type Props = {+ className?: string;+};++function MicIcon({ className }: Props) {+ return (+ <svg+ xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24"+ fill="none"+ className={className}+ aria-hidden="true"+ >+ <path+ d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z"+ stroke="currentColor"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ <path+ d="M19 10v2a7 7 0 0 1-14 0v-2"+ stroke="currentColor"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ <line+ x1="12"+ x2="12"+ y1="19"+ y2="22"+ stroke="currentColor"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ </svg>+ );+}++export { MicIcon };diff --git a/skyvern-frontend/src/hooks/useSpeechToText.test.ts b/skyvern-frontend/src/hooks/useSpeechToText.test.tsindex 7992680c7..6ac93ea49 100644--- a/skyvern-frontend/src/hooks/useSpeechToText.test.ts+++ b/skyvern-frontend/src/hooks/useSpeechToText.test.ts@@ -119,6 +119,20 @@ describe("isSpeechRecognitionSupported", () => {});});+const mockGetUserMedia = vi.fn().mockResolvedValue({+ getTracks: () => [{ stop: vi.fn() }],+});++async function startSpeech(+ start: () => void,+): Promise<MockSpeechRecognition | null> {+ await act(async () => {+ start();+ await Promise.resolve();+ });+ return MockSpeechRecognition.lastInstance;+}+describe("useSpeechToText", () => {const trackStop = vi.fn();@@ -127,6 +141,9 @@ describe("useSpeechToText", () => {MockSpeechRecognition.lastInstance = null;MockMediaRecorder.lastInstance = null;trackStop.mockClear();+ mockGetUserMedia.mockResolvedValue({+ getTracks: () => [{ stop: trackStop }],+ });(window as { SpeechRecognition?: typeof MockSpeechRecognition }).SpeechRecognition = MockSpeechRecognition;@@ -136,13 +153,7 @@ describe("useSpeechToText", () => {});Object.defineProperty(navigator, "mediaDevices", {configurable: true,- value: {- getUserMedia: vi.fn(() =>- Promise.resolve({- getTracks: () => [{ stop: trackStop }],- } as unknown as MediaStream),- ),- },+ value: { getUserMedia: mockGetUserMedia },});});@@ -162,26 +173,25 @@ describe("useSpeechToText", () => {expect(result.current.isSupported).toBe(true);});- it("starts and stops recognition", () => {+ it("starts and stops recognition", async () => {const { result } = renderHook(() =>useSpeechToText({ onTranscript: vi.fn() }),);- act(() => {- result.current.start();- });+ const instance = await startSpeech(() => result.current.start());+ expect(mockGetUserMedia).toHaveBeenCalledWith({ audio: true });expect(result.current.isListening).toBe(true);- expect(MockSpeechRecognition.lastInstance?.start).toHaveBeenCalled();- expect(MockSpeechRecognition.lastInstance?.continuous).toBe(true);- expect(MockSpeechRecognition.lastInstance?.interimResults).toBe(true);+ expect(instance?.start).toHaveBeenCalled();+ expect(instance?.continuous).toBe(true);+ expect(instance?.interimResults).toBe(true);act(() => {result.current.stop();});expect(result.current.isListening).toBe(false);- expect(MockSpeechRecognition.lastInstance?.stop).toHaveBeenCalled();+ expect(instance?.stop).toHaveBeenCalled();});it("captures an audio blob while dictating", async () => {@@ -250,14 +260,12 @@ describe("useSpeechToText", () => {expect(trackStop).toHaveBeenCalled();});- it("toggles listening on and off", () => {+ it("toggles listening on and off", async () => {const { result } = renderHook(() =>useSpeechToText({ onTranscript: vi.fn() }),);- act(() => {- result.current.toggle();- });+ await startSpeech(() => result.current.toggle());expect(result.current.isListening).toBe(true);act(() => {@@ -266,7 +274,7 @@ describe("useSpeechToText", () => {expect(result.current.isListening).toBe(false);});- it("appends final transcript chunks with spacing", () => {+ it("appends final transcript chunks with spacing", async () => {const onTranscript = vi.fn();const { result } = renderHook(() =>useSpeechToText({@@ -275,9 +283,7 @@ describe("useSpeechToText", () => {}),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitResult("hello world");@@ -294,7 +300,7 @@ describe("useSpeechToText", () => {);});- it("starts from an empty base text", () => {+ it("starts from an empty base text", async () => {const onTranscript = vi.fn();const { result } = renderHook(() =>useSpeechToText({@@ -303,9 +309,7 @@ describe("useSpeechToText", () => {}),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitResult("first phrase");@@ -313,8 +317,11 @@ describe("useSpeechToText", () => {expect(onTranscript).toHaveBeenLastCalledWith("first phrase");});- it("surfaces permission errors", () => {+ it("surfaces permission errors when microphone access is denied", async () => {const onError = vi.fn();+ mockGetUserMedia.mockRejectedValue(+ new DOMException("Permission denied", "NotAllowedError"),+ );const { result } = renderHook(() =>useSpeechToText({onTranscript: vi.fn(),@@ -322,9 +329,46 @@ describe("useSpeechToText", () => {}),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());++ expect(onError).toHaveBeenCalledWith(+ "Microphone access was denied. Allow microphone permission and try again.",+ );+ expect(result.current.isListening).toBe(false);+ expect(MockSpeechRecognition.lastInstance).toBeNull();+ });++ it("surfaces no-microphone errors from getUserMedia", async () => {+ const onError = vi.fn();+ mockGetUserMedia.mockRejectedValue(+ new DOMException("Requested device not found", "NotFoundError"),+ );+ const { result } = renderHook(() =>+ useSpeechToText({+ onTranscript: vi.fn(),+ onError,+ }),+ );++ await startSpeech(() => result.current.start());++ expect(onError).toHaveBeenCalledWith(+ "No microphone was found. Check your device and try again.",+ );+ expect(result.current.isListening).toBe(false);+ expect(MockSpeechRecognition.lastInstance).toBeNull();+ });++ it("surfaces speech recognition permission errors", async () => {+ const onError = vi.fn();+ const { result } = renderHook(() =>+ useSpeechToText({+ onTranscript: vi.fn(),+ onError,+ }),+ );++ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitError("not-allowed");@@ -336,7 +380,7 @@ describe("useSpeechToText", () => {expect(result.current.isListening).toBe(false);});- it("stops when enabled becomes false", () => {+ it("stops when enabled becomes false", async () => {const { result, rerender } = renderHook(({ enabled }: { enabled: boolean }) =>useSpeechToText({@@ -346,9 +390,7 @@ describe("useSpeechToText", () => {{ initialProps: { enabled: true } },);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());expect(result.current.isListening).toBe(true);rerender({ enabled: false });@@ -356,14 +398,12 @@ describe("useSpeechToText", () => {expect(result.current.isListening).toBe(false);});- it("cleans up active recognition on unmount", () => {+ it("cleans up active recognition on unmount", async () => {const { result, unmount } = renderHook(() =>useSpeechToText({ onTranscript: vi.fn() }),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());const instance = MockSpeechRecognition.lastInstance;unmount();@@ -371,7 +411,7 @@ describe("useSpeechToText", () => {expect(instance?.abort).toHaveBeenCalled();});- it("updates transcript with interim results", () => {+ it("updates transcript with interim results", async () => {const onTranscript = vi.fn();const { result } = renderHook(() =>useSpeechToText({@@ -380,9 +420,7 @@ describe("useSpeechToText", () => {}),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitResult("hel", false);@@ -390,14 +428,12 @@ describe("useSpeechToText", () => {expect(onTranscript).toHaveBeenLastCalledWith("base hel");});- it("pulses isHearingSpeech while interim speech is detected", () => {+ it("pulses isHearingSpeech while interim speech is detected", async () => {const { result } = renderHook(() =>useSpeechToText({ onTranscript: vi.fn() }),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitResult("hel", false);@@ -410,14 +446,12 @@ describe("useSpeechToText", () => {expect(result.current.isHearingSpeech).toBe(false);});- it("clears isHearingSpeech when dictation stops", () => {+ it("clears isHearingSpeech when dictation stops", async () => {const { result } = renderHook(() =>useSpeechToText({ onTranscript: vi.fn() }),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitResult("hel", false);@@ -430,14 +464,12 @@ describe("useSpeechToText", () => {expect(result.current.isHearingSpeech).toBe(false);});- it("restarts recognition when the browser ends the session while listening", () => {+ it("restarts recognition when the browser ends the session while listening", async () => {const { result } = renderHook(() =>useSpeechToText({ onTranscript: vi.fn() }),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());const instance = MockSpeechRecognition.lastInstance;expect(instance?.start).toHaveBeenCalledTimes(1);@@ -456,7 +488,7 @@ describe("useSpeechToText", () => {expect(result.current.isListening).toBe(true);});- it("stops listening after too many rapid auto-restarts", () => {+ it("stops listening after too many rapid auto-restarts", async () => {const onError = vi.fn();const { result } = renderHook(() =>useSpeechToText({@@ -465,9 +497,7 @@ describe("useSpeechToText", () => {}),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());const instance = MockSpeechRecognition.lastInstance;expect(instance).not.toBeNull();@@ -485,7 +515,7 @@ describe("useSpeechToText", () => {);});- it("ignores no-speech errors while listening", () => {+ it("ignores no-speech errors while listening", async () => {const onError = vi.fn();const { result } = renderHook(() =>useSpeechToText({@@ -494,9 +524,7 @@ describe("useSpeechToText", () => {}),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitError("no-speech");@@ -522,7 +550,7 @@ describe("useSpeechToText", () => {expect(MockSpeechRecognition.lastInstance).toBeNull();});- it("uses cached finalized chunks for results before resultIndex", () => {+ it("uses cached finalized chunks for results before resultIndex", async () => {const onTranscript = vi.fn();const { result } = renderHook(() =>useSpeechToText({@@ -531,9 +559,7 @@ describe("useSpeechToText", () => {}),);- act(() => {- result.current.start();- });+ await startSpeech(() => result.current.start());act(() => {MockSpeechRecognition.lastInstance?.emitResult("hello world");diff --git a/skyvern-frontend/src/hooks/useSpeechToText.ts b/skyvern-frontend/src/hooks/useSpeechToText.tsindex f6c0ec442..522adde7c 100644--- a/skyvern-frontend/src/hooks/useSpeechToText.ts+++ b/skyvern-frontend/src/hooks/useSpeechToText.ts@@ -163,6 +163,30 @@ function errorMessageForCode(error: string): string {return ERROR_MESSAGES[error] ?? "Voice input failed. Try again.";}+function microphonePermissionErrorMessage(error: unknown): string {+ if (error instanceof DOMException) {+ if (error.name === "NotAllowedError" || error.name === "SecurityError") {+ return ERROR_MESSAGES["not-allowed"]!;+ }+ if (error.name === "NotFoundError") {+ return ERROR_MESSAGES["audio-capture"]!;+ }+ }+ return "Voice input failed to start. Try again.";+}++async function requestMicrophonePermission(): Promise<void> {+ if (+ typeof navigator === "undefined" ||+ !navigator.mediaDevices?.getUserMedia+ ) {+ return;+ }++ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });+ stream.getTracks().forEach((track) => track.stop());+}+export function useSpeechToText(options: UseSpeechToTextOptions,): UseSpeechToTextResult {@@ -197,6 +221,7 @@ export function useSpeechToText(const audioStopResolverRef = useRef<((blob: Blob | null) => void) | null>(null,);+ const isStartingRef = useRef(false);const onTranscriptRef = useRef(onTranscript);const onErrorRef = useRef(onError);@@ -382,13 +407,13 @@ export function useSpeechToText(teardownRecognition,]);- const start = useCallback(() => {- if (!isSupported || !enabledRef.current || isListeningRef.current) {- return;- }-+ const beginRecognition = useCallback(() => {const SpeechRecognitionCtor = getSpeechRecognitionConstructor();- if (!SpeechRecognitionCtor) {+ if (+ !SpeechRecognitionCtor ||+ !enabledRef.current ||+ isListeningRef.current+ ) {return;}@@ -512,7 +537,6 @@ export function useSpeechToText(}, [clearHearingSpeech,clearRestartTimeout,- isSupported,lang,markHearingSpeech,startAudioCapture,@@ -520,6 +544,35 @@ export function useSpeechToText(teardownRecognition,]);+ const start = useCallback(() => {+ if (+ !isSupported ||+ !enabledRef.current ||+ isListeningRef.current ||+ isStartingRef.current+ ) {+ return;+ }++ isStartingRef.current = true;++ void (async () => {+ try {+ await requestMicrophonePermission();++ if (!enabledRef.current || isListeningRef.current) {+ return;+ }++ beginRecognition();+ } catch (error) {+ onErrorRef.current?.(microphonePermissionErrorMessage(error));+ } finally {+ isStartingRef.current = false;+ }+ })();+ }, [beginRecognition, isSupported]);+const toggle = useCallback(() => {if (isListeningRef.current) {void stop();