Skip to content

Automate browser based workflows with AI

fix(SKY-10700): enable browser mic permission prompt for voice input (#6601)

Cindy Lic0d3d0d3ad2c
diff --git a/skyvern-frontend/src/components/SpeechInputButton.tsx b/skyvern-frontend/src/components/SpeechInputButton.tsx
index 6a6c6125a..e814721ec 100644
--- a/skyvern-frontend/src/components/SpeechInputButton.tsx
+++ b/skyvern-frontend/src/components/SpeechInputButton.tsx
@@ -1,3 +1,4 @@
+import { MicIcon } from "@/components/icons/MicIcon";
import {
Tooltip,
TooltipContent,
@@ -6,24 +7,6 @@ import {
} from "@/components/ui/tooltip";
import { cn } from "@/util/utils";
-function MicIcon({ className }: { className?: string }) {
- return (
- <svg
- xmlns="http://www.w3.org/2000/svg"
- viewBox="0 0 15 15"
- fill="currentColor"
- className={className}
- aria-hidden="true"
- >
- <path
- fillRule="evenodd"
- clipRule="evenodd"
- d="M7.5 1.25a2.25 2.25 0 0 0-2.25 2.25v3.5a2.25 2.25 0 0 0 4.5 0V3.5A2.25 2.25 0 0 0 7.5 1.25Zm-3.75 5.25a3.75 3.75 0 0 0 7.5 0H10a4.75 4.75 0 0 1-9.5 0H3.75ZM7.5 10.25a.75.75 0 0 1 .75.75v1.5h1.25a.75.75 0 0 1 0 1.5H5.5a.75.75 0 0 1 0-1.5h1.25v-1.5A.75.75 0 0 1 7.5 10.25Z"
- />
- </svg>
- );
-}
-
type SpeechInputButtonProps = {
isSupported: boolean;
isListening: boolean;
diff --git a/skyvern-frontend/src/components/icons/MicIcon.tsx b/skyvern-frontend/src/components/icons/MicIcon.tsx
new file mode 100644
index 000000000..77e9b3e9f
--- /dev/null
+++ b/skyvern-frontend/src/components/icons/MicIcon.tsx
@@ -0,0 +1,42 @@
+type Props = {
+ className?: string;
+};
+
+function MicIcon({ className }: Props) {
+ return (
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ className={className}
+ aria-hidden="true"
+ >
+ <path
+ d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z"
+ stroke="currentColor"
+ strokeWidth="2"
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ />
+ <path
+ d="M19 10v2a7 7 0 0 1-14 0v-2"
+ stroke="currentColor"
+ strokeWidth="2"
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ />
+ <line
+ x1="12"
+ x2="12"
+ y1="19"
+ y2="22"
+ stroke="currentColor"
+ strokeWidth="2"
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ />
+ </svg>
+ );
+}
+
+export { MicIcon };
diff --git a/skyvern-frontend/src/hooks/useSpeechToText.test.ts b/skyvern-frontend/src/hooks/useSpeechToText.test.ts
index 7992680c7..6ac93ea49 100644
--- a/skyvern-frontend/src/hooks/useSpeechToText.test.ts
+++ b/skyvern-frontend/src/hooks/useSpeechToText.test.ts
@@ -119,6 +119,20 @@ describe("isSpeechRecognitionSupported", () => {
});
});
+const mockGetUserMedia = vi.fn().mockResolvedValue({
+ getTracks: () => [{ stop: vi.fn() }],
+});
+
+async function startSpeech(
+ start: () => void,
+): Promise<MockSpeechRecognition | null> {
+ await act(async () => {
+ start();
+ await Promise.resolve();
+ });
+ return MockSpeechRecognition.lastInstance;
+}
+
describe("useSpeechToText", () => {
const trackStop = vi.fn();
@@ -127,6 +141,9 @@ describe("useSpeechToText", () => {
MockSpeechRecognition.lastInstance = null;
MockMediaRecorder.lastInstance = null;
trackStop.mockClear();
+ mockGetUserMedia.mockResolvedValue({
+ getTracks: () => [{ stop: trackStop }],
+ });
(
window as { SpeechRecognition?: typeof MockSpeechRecognition }
).SpeechRecognition = MockSpeechRecognition;
@@ -136,13 +153,7 @@ describe("useSpeechToText", () => {
});
Object.defineProperty(navigator, "mediaDevices", {
configurable: true,
- value: {
- getUserMedia: vi.fn(() =>
- Promise.resolve({
- getTracks: () => [{ stop: trackStop }],
- } as unknown as MediaStream),
- ),
- },
+ value: { getUserMedia: mockGetUserMedia },
});
});
@@ -162,26 +173,25 @@ describe("useSpeechToText", () => {
expect(result.current.isSupported).toBe(true);
});
- it("starts and stops recognition", () => {
+ it("starts and stops recognition", async () => {
const { result } = renderHook(() =>
useSpeechToText({ onTranscript: vi.fn() }),
);
- act(() => {
- result.current.start();
- });
+ const instance = await startSpeech(() => result.current.start());
+ expect(mockGetUserMedia).toHaveBeenCalledWith({ audio: true });
expect(result.current.isListening).toBe(true);
- expect(MockSpeechRecognition.lastInstance?.start).toHaveBeenCalled();
- expect(MockSpeechRecognition.lastInstance?.continuous).toBe(true);
- expect(MockSpeechRecognition.lastInstance?.interimResults).toBe(true);
+ expect(instance?.start).toHaveBeenCalled();
+ expect(instance?.continuous).toBe(true);
+ expect(instance?.interimResults).toBe(true);
act(() => {
result.current.stop();
});
expect(result.current.isListening).toBe(false);
- expect(MockSpeechRecognition.lastInstance?.stop).toHaveBeenCalled();
+ expect(instance?.stop).toHaveBeenCalled();
});
it("captures an audio blob while dictating", async () => {
@@ -250,14 +260,12 @@ describe("useSpeechToText", () => {
expect(trackStop).toHaveBeenCalled();
});
- it("toggles listening on and off", () => {
+ it("toggles listening on and off", async () => {
const { result } = renderHook(() =>
useSpeechToText({ onTranscript: vi.fn() }),
);
- act(() => {
- result.current.toggle();
- });
+ await startSpeech(() => result.current.toggle());
expect(result.current.isListening).toBe(true);
act(() => {
@@ -266,7 +274,7 @@ describe("useSpeechToText", () => {
expect(result.current.isListening).toBe(false);
});
- it("appends final transcript chunks with spacing", () => {
+ it("appends final transcript chunks with spacing", async () => {
const onTranscript = vi.fn();
const { result } = renderHook(() =>
useSpeechToText({
@@ -275,9 +283,7 @@ describe("useSpeechToText", () => {
}),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitResult("hello world");
@@ -294,7 +300,7 @@ describe("useSpeechToText", () => {
);
});
- it("starts from an empty base text", () => {
+ it("starts from an empty base text", async () => {
const onTranscript = vi.fn();
const { result } = renderHook(() =>
useSpeechToText({
@@ -303,9 +309,7 @@ describe("useSpeechToText", () => {
}),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitResult("first phrase");
@@ -313,8 +317,11 @@ describe("useSpeechToText", () => {
expect(onTranscript).toHaveBeenLastCalledWith("first phrase");
});
- it("surfaces permission errors", () => {
+ it("surfaces permission errors when microphone access is denied", async () => {
const onError = vi.fn();
+ mockGetUserMedia.mockRejectedValue(
+ new DOMException("Permission denied", "NotAllowedError"),
+ );
const { result } = renderHook(() =>
useSpeechToText({
onTranscript: vi.fn(),
@@ -322,9 +329,46 @@ describe("useSpeechToText", () => {
}),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
+
+ expect(onError).toHaveBeenCalledWith(
+ "Microphone access was denied. Allow microphone permission and try again.",
+ );
+ expect(result.current.isListening).toBe(false);
+ expect(MockSpeechRecognition.lastInstance).toBeNull();
+ });
+
+ it("surfaces no-microphone errors from getUserMedia", async () => {
+ const onError = vi.fn();
+ mockGetUserMedia.mockRejectedValue(
+ new DOMException("Requested device not found", "NotFoundError"),
+ );
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ onError,
+ }),
+ );
+
+ await startSpeech(() => result.current.start());
+
+ expect(onError).toHaveBeenCalledWith(
+ "No microphone was found. Check your device and try again.",
+ );
+ expect(result.current.isListening).toBe(false);
+ expect(MockSpeechRecognition.lastInstance).toBeNull();
+ });
+
+ it("surfaces speech recognition permission errors", async () => {
+ const onError = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ onError,
+ }),
+ );
+
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitError("not-allowed");
@@ -336,7 +380,7 @@ describe("useSpeechToText", () => {
expect(result.current.isListening).toBe(false);
});
- it("stops when enabled becomes false", () => {
+ it("stops when enabled becomes false", async () => {
const { result, rerender } = renderHook(
({ enabled }: { enabled: boolean }) =>
useSpeechToText({
@@ -346,9 +390,7 @@ describe("useSpeechToText", () => {
{ initialProps: { enabled: true } },
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
expect(result.current.isListening).toBe(true);
rerender({ enabled: false });
@@ -356,14 +398,12 @@ describe("useSpeechToText", () => {
expect(result.current.isListening).toBe(false);
});
- it("cleans up active recognition on unmount", () => {
+ it("cleans up active recognition on unmount", async () => {
const { result, unmount } = renderHook(() =>
useSpeechToText({ onTranscript: vi.fn() }),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
const instance = MockSpeechRecognition.lastInstance;
unmount();
@@ -371,7 +411,7 @@ describe("useSpeechToText", () => {
expect(instance?.abort).toHaveBeenCalled();
});
- it("updates transcript with interim results", () => {
+ it("updates transcript with interim results", async () => {
const onTranscript = vi.fn();
const { result } = renderHook(() =>
useSpeechToText({
@@ -380,9 +420,7 @@ describe("useSpeechToText", () => {
}),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitResult("hel", false);
@@ -390,14 +428,12 @@ describe("useSpeechToText", () => {
expect(onTranscript).toHaveBeenLastCalledWith("base hel");
});
- it("pulses isHearingSpeech while interim speech is detected", () => {
+ it("pulses isHearingSpeech while interim speech is detected", async () => {
const { result } = renderHook(() =>
useSpeechToText({ onTranscript: vi.fn() }),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitResult("hel", false);
@@ -410,14 +446,12 @@ describe("useSpeechToText", () => {
expect(result.current.isHearingSpeech).toBe(false);
});
- it("clears isHearingSpeech when dictation stops", () => {
+ it("clears isHearingSpeech when dictation stops", async () => {
const { result } = renderHook(() =>
useSpeechToText({ onTranscript: vi.fn() }),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitResult("hel", false);
@@ -430,14 +464,12 @@ describe("useSpeechToText", () => {
expect(result.current.isHearingSpeech).toBe(false);
});
- it("restarts recognition when the browser ends the session while listening", () => {
+ it("restarts recognition when the browser ends the session while listening", async () => {
const { result } = renderHook(() =>
useSpeechToText({ onTranscript: vi.fn() }),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
const instance = MockSpeechRecognition.lastInstance;
expect(instance?.start).toHaveBeenCalledTimes(1);
@@ -456,7 +488,7 @@ describe("useSpeechToText", () => {
expect(result.current.isListening).toBe(true);
});
- it("stops listening after too many rapid auto-restarts", () => {
+ it("stops listening after too many rapid auto-restarts", async () => {
const onError = vi.fn();
const { result } = renderHook(() =>
useSpeechToText({
@@ -465,9 +497,7 @@ describe("useSpeechToText", () => {
}),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
const instance = MockSpeechRecognition.lastInstance;
expect(instance).not.toBeNull();
@@ -485,7 +515,7 @@ describe("useSpeechToText", () => {
);
});
- it("ignores no-speech errors while listening", () => {
+ it("ignores no-speech errors while listening", async () => {
const onError = vi.fn();
const { result } = renderHook(() =>
useSpeechToText({
@@ -494,9 +524,7 @@ describe("useSpeechToText", () => {
}),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitError("no-speech");
@@ -522,7 +550,7 @@ describe("useSpeechToText", () => {
expect(MockSpeechRecognition.lastInstance).toBeNull();
});
- it("uses cached finalized chunks for results before resultIndex", () => {
+ it("uses cached finalized chunks for results before resultIndex", async () => {
const onTranscript = vi.fn();
const { result } = renderHook(() =>
useSpeechToText({
@@ -531,9 +559,7 @@ describe("useSpeechToText", () => {
}),
);
- act(() => {
- result.current.start();
- });
+ await startSpeech(() => result.current.start());
act(() => {
MockSpeechRecognition.lastInstance?.emitResult("hello world");
diff --git a/skyvern-frontend/src/hooks/useSpeechToText.ts b/skyvern-frontend/src/hooks/useSpeechToText.ts
index f6c0ec442..522adde7c 100644
--- a/skyvern-frontend/src/hooks/useSpeechToText.ts
+++ b/skyvern-frontend/src/hooks/useSpeechToText.ts
@@ -163,6 +163,30 @@ function errorMessageForCode(error: string): string {
return ERROR_MESSAGES[error] ?? "Voice input failed. Try again.";
}
+function microphonePermissionErrorMessage(error: unknown): string {
+ if (error instanceof DOMException) {
+ if (error.name === "NotAllowedError" || error.name === "SecurityError") {
+ return ERROR_MESSAGES["not-allowed"]!;
+ }
+ if (error.name === "NotFoundError") {
+ return ERROR_MESSAGES["audio-capture"]!;
+ }
+ }
+ return "Voice input failed to start. Try again.";
+}
+
+async function requestMicrophonePermission(): Promise<void> {
+ if (
+ typeof navigator === "undefined" ||
+ !navigator.mediaDevices?.getUserMedia
+ ) {
+ return;
+ }
+
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ stream.getTracks().forEach((track) => track.stop());
+}
+
export function useSpeechToText(
options: UseSpeechToTextOptions,
): UseSpeechToTextResult {
@@ -197,6 +221,7 @@ export function useSpeechToText(
const audioStopResolverRef = useRef<((blob: Blob | null) => void) | null>(
null,
);
+ const isStartingRef = useRef(false);
const onTranscriptRef = useRef(onTranscript);
const onErrorRef = useRef(onError);
@@ -382,13 +407,13 @@ export function useSpeechToText(
teardownRecognition,
]);
- const start = useCallback(() => {
- if (!isSupported || !enabledRef.current || isListeningRef.current) {
- return;
- }
-
+ const beginRecognition = useCallback(() => {
const SpeechRecognitionCtor = getSpeechRecognitionConstructor();
- if (!SpeechRecognitionCtor) {
+ if (
+ !SpeechRecognitionCtor ||
+ !enabledRef.current ||
+ isListeningRef.current
+ ) {
return;
}
@@ -512,7 +537,6 @@ export function useSpeechToText(
}, [
clearHearingSpeech,
clearRestartTimeout,
- isSupported,
lang,
markHearingSpeech,
startAudioCapture,
@@ -520,6 +544,35 @@ export function useSpeechToText(
teardownRecognition,
]);
+ const start = useCallback(() => {
+ if (
+ !isSupported ||
+ !enabledRef.current ||
+ isListeningRef.current ||
+ isStartingRef.current
+ ) {
+ return;
+ }
+
+ isStartingRef.current = true;
+
+ void (async () => {
+ try {
+ await requestMicrophonePermission();
+
+ if (!enabledRef.current || isListeningRef.current) {
+ return;
+ }
+
+ beginRecognition();
+ } catch (error) {
+ onErrorRef.current?.(microphonePermissionErrorMessage(error));
+ } finally {
+ isStartingRef.current = false;
+ }
+ })();
+ }, [beginRecognition, isSupported]);
+
const toggle = useCallback(() => {
if (isListeningRef.current) {
void stop();