🙌 OpenHands: AI-Driven Development
feat(chat): add optional voice dictation to the chat input (#17804)
Vasco Schiavoa8c8fb377761
diff --git a/__tests__/components/chat/chat-dictation-button.test.tsx b/__tests__/components/chat/chat-dictation-button.test.tsxnew file mode 100644index 0000000000..54d4690f93--- /dev/null+++ b/__tests__/components/chat/chat-dictation-button.test.tsx@@ -0,0 +1,208 @@+import { act, render, screen, waitFor } from "@testing-library/react";+import userEvent from "@testing-library/user-event";+import { afterEach, describe, expect, it, vi } from "vitest";+import { ChatDictationButton } from "#/components/features/chat/chat-dictation-button";+import { I18nKey } from "#/i18n/declaration";+import { displayErrorToast } from "#/utils/custom-toast-handlers";+import { writeTranscriptionEndpoint } from "#/utils/transcription-endpoint-storage";++vi.mock("#/utils/custom-toast-handlers", () => ({+ displayErrorToast: vi.fn(),+}));++class FakeSpeechRecognition {+ static instance: FakeSpeechRecognition;++ continuous = false;++ interimResults = true;++ onresult: ((event: unknown) => void) | null = null;++ onerror: ((event: { error: string }) => void) | null = null;++ onend: (() => void) | null = null;++ constructor() {+ FakeSpeechRecognition.instance = this;+ }++ start = vi.fn();++ stop = vi.fn(() => this.onend?.());++ abort = vi.fn();+}++class FakeMediaRecorder {+ mimeType = "audio/webm";++ ondataavailable: ((event: { data: Blob }) => void) | null = null;++ onstop: (() => void) | null = null;++ start = vi.fn();++ stop = vi.fn(() => {+ this.ondataavailable?.({ data: new Blob(["audio"]) });+ this.onstop?.();+ });+}++afterEach(() => {+ vi.unstubAllGlobals();+ vi.restoreAllMocks();+ localStorage.clear();+ sessionStorage.clear();+});++describe("ChatDictationButton", () => {+ it("renders nothing when the browser cannot dictate", () => {+ render(<ChatDictationButton onTranscript={vi.fn()} />);++ expect(screen.queryByTestId("chat-dictation-button")).toBeNull();+ });++ it("dictates with browser speech recognition until stopped", async () => {+ vi.stubGlobal("webkitSpeechRecognition", FakeSpeechRecognition);+ const user = userEvent.setup();+ const onTranscript = vi.fn();+ render(<ChatDictationButton onTranscript={onTranscript} />);++ await user.click(screen.getByTestId("chat-dictation-button"));+ FakeSpeechRecognition.instance.onresult?.({+ results: [[{ transcript: "fix the" }], [{ transcript: "login bug" }]],+ });+ const button = screen.getByTestId("chat-dictation-button");+ expect(button).toHaveAttribute(+ "aria-label",+ I18nKey.CHAT_INTERFACE$STOP_DICTATION,+ );+ await user.click(button);++ expect(onTranscript).toHaveBeenCalledWith("fix the login bug");+ expect(button).toHaveAttribute(+ "aria-label",+ I18nKey.CHAT_INTERFACE$START_DICTATION,+ );+ });++ it("transcribes recorded audio with the configured endpoint", async () => {+ writeTranscriptionEndpoint({+ baseUrl: " http://localhost:9000/v1/ ",+ apiKey: "secret",+ });+ vi.stubGlobal("MediaRecorder", FakeMediaRecorder);+ vi.stubGlobal("navigator", {+ mediaDevices: {+ getUserMedia: vi.fn().mockResolvedValue({ getTracks: () => [] }),+ },+ });+ const fetchSpy = vi+ .spyOn(globalThis, "fetch")+ .mockResolvedValue(Response.json({ text: " hello world " }));+ const user = userEvent.setup();+ const onTranscript = vi.fn();+ render(<ChatDictationButton onTranscript={onTranscript} />);++ await user.click(screen.getByTestId("chat-dictation-button"));+ await user.click(screen.getByTestId("chat-dictation-button"));++ await waitFor(() =>+ expect(onTranscript).toHaveBeenCalledWith("hello world"),+ );+ const [url, init] = fetchSpy.mock.calls[0];+ expect(url).toBe("http://localhost:9000/v1/audio/transcriptions");+ expect(init?.headers).toEqual({ Authorization: "Bearer secret" });+ expect(JSON.stringify({ ...localStorage })).not.toContain("secret");+ expect((init?.body as FormData).get("model")).toBe("whisper-1");+ });++ it("shows an error toast when microphone access is denied", async () => {+ writeTranscriptionEndpoint({ baseUrl: "http://localhost:9000/v1" });+ vi.stubGlobal("MediaRecorder", FakeMediaRecorder);+ vi.stubGlobal("navigator", {+ mediaDevices: {+ getUserMedia: vi.fn().mockRejectedValue(new Error("NotAllowedError")),+ },+ });+ const user = userEvent.setup();+ render(<ChatDictationButton onTranscript={vi.fn()} />);++ await user.click(screen.getByTestId("chat-dictation-button"));++ expect(displayErrorToast).toHaveBeenCalledWith(+ I18nKey.CHAT_INTERFACE$DICTATION_FAILED,+ );+ });++ it("ignores clicks while the permission prompt is open", async () => {+ writeTranscriptionEndpoint({ baseUrl: "http://localhost:9000/v1" });+ vi.stubGlobal("MediaRecorder", FakeMediaRecorder);+ let grant: (stream: MediaStream) => void = () => {};+ const getUserMedia = vi.fn(+ () =>+ new Promise<MediaStream>((resolve) => {+ grant = resolve;+ }),+ );+ vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } });+ const user = userEvent.setup();+ render(<ChatDictationButton onTranscript={vi.fn()} />);++ await user.click(screen.getByTestId("chat-dictation-button"));+ await user.click(screen.getByTestId("chat-dictation-button"));++ expect(getUserMedia).toHaveBeenCalledOnce();+ await act(async () =>+ grant({ getTracks: () => [] } as unknown as MediaStream),+ );+ expect(screen.getByTestId("chat-dictation-button")).toHaveAttribute(+ "aria-pressed",+ "true",+ );+ });++ it("can still stop a recording after the input turns read-only", async () => {+ vi.stubGlobal("webkitSpeechRecognition", FakeSpeechRecognition);+ const user = userEvent.setup();+ const { rerender } = render(<ChatDictationButton onTranscript={vi.fn()} />);++ await user.click(screen.getByTestId("chat-dictation-button"));+ rerender(<ChatDictationButton onTranscript={vi.fn()} disabled />);++ expect(screen.getByTestId("chat-dictation-button")).toBeEnabled();+ await user.click(screen.getByTestId("chat-dictation-button"));+ expect(FakeSpeechRecognition.instance.stop).toHaveBeenCalledOnce();+ expect(screen.getByTestId("chat-dictation-button")).toBeDisabled();+ });++ it("releases the microphone when unmounted during the permission prompt", async () => {+ writeTranscriptionEndpoint({ baseUrl: "http://localhost:9000/v1" });+ const MediaRecorderSpy = vi.fn();+ vi.stubGlobal("MediaRecorder", MediaRecorderSpy);+ let grant: (stream: MediaStream) => void = () => {};+ vi.stubGlobal("navigator", {+ mediaDevices: {+ getUserMedia: () =>+ new Promise<MediaStream>((resolve) => {+ grant = resolve;+ }),+ },+ });+ const stopTrack = vi.fn();+ const user = userEvent.setup();+ const { unmount } = render(<ChatDictationButton onTranscript={vi.fn()} />);++ await user.click(screen.getByTestId("chat-dictation-button"));+ unmount();+ await act(async () =>+ grant({+ getTracks: () => [{ stop: stopTrack }],+ } as unknown as MediaStream),+ );++ expect(stopTrack).toHaveBeenCalledOnce();+ expect(MediaRecorderSpy).not.toHaveBeenCalled();+ });+});diff --git a/__tests__/components/features/chat/utils/chat-input.utils.test.ts b/__tests__/components/features/chat/utils/chat-input.utils.test.tsindex 2eb7e903b2..22d646690a 100644--- a/__tests__/components/features/chat/utils/chat-input.utils.test.ts+++ b/__tests__/components/features/chat/utils/chat-input.utils.test.ts@@ -7,6 +7,7 @@ import {focusContentEditableAtEnd,getClipboardFiles,getTextContent,+ insertTextAtCaret,isContentEmpty,isPastedClipboardImage,normalizePastedFile,@@ -345,3 +346,44 @@ describe("content-editable helpers", () => {expect(focus).toHaveBeenCalledTimes(2);});});++describe("insertTextAtCaret", () => {+ function setup(text: string, caretOffset?: number) {+ const element = document.createElement("div");+ element.textContent = text;+ document.body.appendChild(element);+ window.getSelection()!.removeAllRanges();+ if (caretOffset !== undefined) {+ const range = document.createRange();+ range.setStart(element.firstChild!, caretOffset);+ window.getSelection()!.addRange(range);+ }+ const execCommand = vi.fn();+ Object.defineProperty(document, "execCommand", {+ configurable: true,+ value: execCommand,+ });+ return { element, execCommand };+ }++ afterEach(() => {+ Reflect.deleteProperty(document, "execCommand");+ document.body.replaceChildren();+ });++ it("inserts through the paste path, padding against adjacent words", () => {+ const { element, execCommand } = setup("fix the bug", "fix the ".length);++ insertTextAtCaret(element, "login");++ expect(execCommand).toHaveBeenCalledWith("insertText", false, "login ");+ });++ it("adds a leading space after a word and appends when the caret is outside", () => {+ const { element, execCommand } = setup("hello");++ insertTextAtCaret(element, "world");++ expect(execCommand).toHaveBeenCalledWith("insertText", false, " world");+ });+});diff --git a/src/components/features/chat/chat-dictation-button.tsx b/src/components/features/chat/chat-dictation-button.tsxnew file mode 100644index 0000000000..2feec29278--- /dev/null+++ b/src/components/features/chat/chat-dictation-button.tsx@@ -0,0 +1,67 @@+import { Loader2, Mic, Square } from "lucide-react";+import { useTranslation } from "react-i18next";+import { I18nKey } from "#/i18n/declaration";+import { useDictation, type DictationStatus } from "#/hooks/chat/use-dictation";+import { displayErrorToast } from "#/utils/custom-toast-handlers";+import { chatInputIconButtonClassName } from "#/utils/form-control-classes";+import { cn } from "#/utils/utils";++const STATUS_ICONS: Record<DictationStatus, typeof Mic> = {+ idle: Mic,+ starting: Loader2,+ recording: Square,+ transcribing: Loader2,+};++export interface ChatDictationButtonProps {+ onTranscript: (text: string) => void;+ disabled?: boolean;+}++export function ChatDictationButton({+ onTranscript,+ disabled = false,+}: ChatDictationButtonProps) {+ const { t } = useTranslation("openhands");+ const { status, isSupported, toggle } = useDictation({+ onTranscript,+ onError: () =>+ displayErrorToast(t(I18nKey.CHAT_INTERFACE$DICTATION_FAILED)),+ });++ if (!isSupported) return null;++ const isRecording = status === "recording";+ const isBusy = status === "starting" || status === "transcribing";+ // Stopping stays possible even if the text field turned read-only.+ const isDisabled = !isRecording && (disabled || status !== "idle");+ const Icon = STATUS_ICONS[status];++ return (+ <button+ type="button"+ className={cn(+ chatInputIconButtonClassName,+ "shrink-0 size-8",+ isRecording && "text-danger",+ isDisabled && "cursor-not-allowed text-text-subtle",+ )}+ aria-label={t(+ isRecording+ ? I18nKey.CHAT_INTERFACE$STOP_DICTATION+ : I18nKey.CHAT_INTERFACE$START_DICTATION,+ )}+ aria-pressed={isRecording}+ aria-busy={isBusy}+ data-testid="chat-dictation-button"+ onClick={toggle}+ disabled={isDisabled}+ >+ <Icon+ className={cn("size-4", isBusy && "animate-spin")}+ fill={isRecording ? "currentColor" : "none"}+ aria-hidden+ />+ </button>+ );+}diff --git a/src/components/features/chat/components/chat-input-actions.tsx b/src/components/features/chat/components/chat-input-actions.tsxindex de1c3633db..6c1298a136 100644--- a/src/components/features/chat/components/chat-input-actions.tsx+++ b/src/components/features/chat/components/chat-input-actions.tsx@@ -12,6 +12,7 @@ import {import { resolvePickerKind } from "./resolve-picker-kind";import { ChatAddFileButton } from "../chat-add-file-button";import { ChatSendButton } from "../chat-send-button";+import { ChatDictationButton } from "../chat-dictation-button";import { ContextWindowMeter } from "./context-window-meter";import CarretRightFillIcon from "#/icons/carret-right-fill.svg?react";import LessonPlanIcon from "#/icons/lesson-plan.svg?react";@@ -48,6 +49,9 @@ interface ChatInputActionsProps {showButton?: boolean;buttonClassName?: string;handleSubmit?: () => void;+ onDictationTranscript?: (text: string) => void;+ /** Tracks the text field's editability, not submit availability. */+ isDictationDisabled?: boolean;}export function ChatInputActions({@@ -58,6 +62,8 @@ export function ChatInputActions({showButton = true,buttonClassName = "",handleSubmit = () => {},+ onDictationTranscript,+ isDictationDisabled = false,}: ChatInputActionsProps) {const { t } = useTranslation("openhands");const unifiedPauseMutation = useUnifiedPauseConversation();@@ -499,6 +505,12 @@ export function ChatInputActions({/>)}<ContextWindowMeter />+ {onDictationTranscript && (+ <ChatDictationButton+ onTranscript={onDictationTranscript}+ disabled={isDictationDisabled}+ />+ )}{showButton && (<ChatSendButtonbuttonClassName={buttonClassName}diff --git a/src/components/features/chat/components/chat-input-container.tsx b/src/components/features/chat/components/chat-input-container.tsxindex ba7b01872c..ef77a4ab04 100644--- a/src/components/features/chat/components/chat-input-container.tsx+++ b/src/components/features/chat/components/chat-input-container.tsx@@ -7,6 +7,7 @@ import { SlashCommandMenu } from "./slash-command-menu";import { useConversationStore } from "#/stores/conversation-store";import { cn } from "#/utils/utils";import { SlashCommandItem } from "#/hooks/chat/use-slash-command";+import { insertTextAtCaret } from "../utils/chat-input.utils";interface ChatInputContainerProps {chatContainerRef: React.RefObject<HTMLDivElement | null>;@@ -112,6 +113,10 @@ export function ChatInputContainer({showButton={showButton}buttonClassName={buttonClassName}handleSubmit={handleSubmit}+ onDictationTranscript={(text) =>+ insertTextAtCaret(chatInputRef.current, text)+ }+ isDictationDisabled={isNewConversationPending}/></div>);diff --git a/src/components/features/chat/utils/chat-input.utils.ts b/src/components/features/chat/utils/chat-input.utils.tsindex a91dd00dbf..b591875082 100644--- a/src/components/features/chat/utils/chat-input.utils.ts+++ b/src/components/features/chat/utils/chat-input.utils.ts@@ -177,3 +177,39 @@ export const focusContentEditableAtEnd = (selection.addRange(range);ensureCursorVisible(element);};++/**+ * Insert text at the caret (or the end when the caret is elsewhere), padded+ * with spaces so it doesn't fuse with neighbouring words.+ */+export const insertTextAtCaret = (+ element: HTMLElement | null,+ text: string,+): void => {+ if (!element) {+ return;+ }++ const selection = window.getSelection();+ if (!selection) {+ return;+ }+ if (selection.rangeCount === 0 || !element.contains(selection.anchorNode)) {+ focusContentEditableAtEnd(element);+ } else {+ element.focus();+ }++ const caret = selection.getRangeAt(0);+ const context = document.createRange();+ context.selectNodeContents(element);+ context.setEnd(caret.startContainer, caret.startOffset);+ const lead = /\S$/.test(context.toString()) ? " " : "";+ context.selectNodeContents(element);+ context.setStart(caret.endContainer, caret.endOffset);+ const trail = /^\S/.test(context.toString()) ? " " : "";++ // Same path as plain-text paste: fires `input`, keeps undo, and is a no-op+ // when the field is read-only.+ document.execCommand("insertText", false, `${lead}${text}${trail}`);+};diff --git a/src/components/features/settings/app-settings/voice-input-settings.tsx b/src/components/features/settings/app-settings/voice-input-settings.tsxnew file mode 100644index 0000000000..d6d9a6625c--- /dev/null+++ b/src/components/features/settings/app-settings/voice-input-settings.tsx@@ -0,0 +1,59 @@+import React from "react";+import { useTranslation } from "react-i18next";+import { SettingsInput } from "#/components/features/settings/settings-input";+import { I18nKey } from "#/i18n/declaration";+import {+ DEFAULT_TRANSCRIPTION_MODEL,+ readTranscriptionEndpoint,+ writeTranscriptionEndpoint,+} from "#/utils/transcription-endpoint-storage";++const TRANSCRIPTION_BASE_URL_PLACEHOLDER = "https://api.openai.com/v1";++/** Saved on change to browser storage, independently of the settings form. */+export function VoiceInputSettings() {+ const { t } = useTranslation("openhands");+ const [endpoint] = React.useState(readTranscriptionEndpoint);++ return (+ <div className="border-t border-border pt-6 mt-2">+ <h3 className="text-lg font-medium mb-2">+ {t(I18nKey.SETTINGS$VOICE_INPUT)}+ </h3>+ <p className="mb-4 text-sm leading-5 text-tertiary-light">+ {t(I18nKey.SETTINGS$VOICE_INPUT_DESCRIPTION)}+ </p>+ <div className="flex flex-col gap-6">+ <SettingsInput+ testId="transcription-base-url-input"+ type="text"+ label={t(I18nKey.SETTINGS$BASE_URL)}+ defaultValue={endpoint.baseUrl}+ placeholder={TRANSCRIPTION_BASE_URL_PLACEHOLDER}+ onChange={(baseUrl) => writeTranscriptionEndpoint({ baseUrl })}+ showOptionalTag+ className="w-full min-w-0"+ />+ <SettingsInput+ testId="transcription-api-key-input"+ type="password"+ label={t(I18nKey.SETTINGS_FORM$API_KEY)}+ defaultValue={endpoint.apiKey}+ onChange={(apiKey) => writeTranscriptionEndpoint({ apiKey })}+ showOptionalTag+ className="w-full min-w-0"+ />+ <SettingsInput+ testId="transcription-model-input"+ type="text"+ label={t(I18nKey.SETTINGS$AGENT_MODEL)}+ defaultValue={endpoint.model}+ placeholder={DEFAULT_TRANSCRIPTION_MODEL}+ onChange={(model) => writeTranscriptionEndpoint({ model })}+ showOptionalTag+ className="w-full min-w-0"+ />+ </div>+ </div>+ );+}diff --git a/src/hooks/chat/use-dictation.ts b/src/hooks/chat/use-dictation.tsnew file mode 100644index 0000000000..b8a1247307--- /dev/null+++ b/src/hooks/chat/use-dictation.ts@@ -0,0 +1,205 @@+import React from "react";+import {+ DEFAULT_TRANSCRIPTION_MODEL,+ readTranscriptionEndpoint,+ type TranscriptionEndpoint,+} from "#/utils/transcription-endpoint-storage";++export type DictationStatus =+ | "idle"+ | "starting"+ | "recording"+ | "transcribing";++// Web Speech API types are not in lib.dom; this is the subset we use.+interface SpeechRecognitionLike {+ continuous: boolean;+ interimResults: boolean;+ onresult:+ | ((event: {+ results: ArrayLike<ArrayLike<{ transcript: string }>>;+ }) => void)+ | null;+ onerror: ((event: { error: string }) => void) | null;+ onend: (() => void) | null;+ start: () => void;+ stop: () => void;+ abort: () => void;+}++type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;++interface DictationSession {+ stop: () => void;+ cancel: () => void;+}++function getSpeechRecognition(): SpeechRecognitionConstructor | undefined {+ if (typeof window === "undefined") return undefined;+ const speechWindow = window as Window & {+ SpeechRecognition?: SpeechRecognitionConstructor;+ webkitSpeechRecognition?: SpeechRecognitionConstructor;+ };+ return speechWindow.SpeechRecognition ?? speechWindow.webkitSpeechRecognition;+}++const canRecordAudio = () =>+ typeof MediaRecorder !== "undefined" &&+ !!navigator.mediaDevices?.getUserMedia;++async function transcribe(+ audio: Blob,+ endpoint: TranscriptionEndpoint,+): Promise<string> {+ // Whisper servers infer the codec from the file extension.+ const extension = audio.type.split(/[/;]/)[1] || "webm";+ const body = new FormData();+ body.append("file", audio, `dictation.${extension}`);+ body.append("model", endpoint.model || DEFAULT_TRANSCRIPTION_MODEL);++ const response = await fetch(+ `${endpoint.baseUrl.replace(/\/+$/, "")}/audio/transcriptions`,+ {+ method: "POST",+ headers: endpoint.apiKey+ ? { Authorization: `Bearer ${endpoint.apiKey}` }+ : undefined,+ body,+ },+ );+ if (!response.ok) {+ throw new Error(`Transcription failed with status ${response.status}`);+ }+ const { text } = await response.json();+ return typeof text === "string" ? text.trim() : "";+}++interface UseDictationOptions {+ onTranscript: (text: string) => void;+ onError: (error: unknown) => void;+}++/**+ * Push-to-toggle dictation. Uses the configured OpenAI-compatible endpoint+ * when set, otherwise the browser's speech recognition.+ */+export function useDictation({ onTranscript, onError }: UseDictationOptions) {+ const [status, setStatus] = React.useState<DictationStatus>("idle");+ const sessionRef = React.useRef<DictationSession | null>(null);+ const mountedRef = React.useRef(true);+ const endpoint = readTranscriptionEndpoint();+ const isSupported = endpoint.baseUrl+ ? canRecordAudio()+ : !!getSpeechRecognition();++ React.useEffect(() => {+ mountedRef.current = true;+ return () => {+ mountedRef.current = false;+ sessionRef.current?.cancel();+ };+ }, []);++ const startSpeechRecognition = (+ Recognition: SpeechRecognitionConstructor,+ ) => {+ const recognition = new Recognition();+ recognition.continuous = true;+ recognition.interimResults = false;++ let transcript = "";+ let cancelled = false;+ recognition.onresult = (event) => {+ transcript = Array.from(event.results, (result) => result[0].transcript)+ .join(" ")+ .trim();+ };+ recognition.onerror = (event) => {+ if (event.error !== "aborted") onError(new Error(event.error));+ };+ recognition.onend = () => {+ sessionRef.current = null;+ setStatus("idle");+ if (!cancelled && transcript) onTranscript(transcript);+ };++ recognition.start();+ sessionRef.current = {+ stop: () => {+ setStatus("transcribing");+ recognition.stop();+ },+ cancel: () => {+ cancelled = true;+ recognition.abort();+ },+ };+ setStatus("recording");+ };++ const startRecording = async (target: TranscriptionEndpoint) => {+ // Enter "starting" before the permission prompt so further clicks are+ // ignored instead of opening a second, untracked stream.+ setStatus("starting");+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });+ if (!mountedRef.current) {+ stream.getTracks().forEach((track) => track.stop());+ return;+ }+ const recorder = new MediaRecorder(stream);+ const chunks: Blob[] = [];+ let cancelled = false;++ recorder.ondataavailable = (event) => chunks.push(event.data);+ recorder.onstop = async () => {+ stream.getTracks().forEach((track) => track.stop());+ sessionRef.current = null;+ if (cancelled) return;++ setStatus("transcribing");+ try {+ const text = await transcribe(+ new Blob(chunks, { type: recorder.mimeType }),+ target,+ );+ if (text) onTranscript(text);+ } catch (error) {+ onError(error);+ } finally {+ setStatus("idle");+ }+ };++ recorder.start();+ sessionRef.current = {+ stop: () => recorder.stop(),+ cancel: () => {+ cancelled = true;+ recorder.stop();+ },+ };+ setStatus("recording");+ };++ const toggle = async () => {+ if (sessionRef.current) {+ sessionRef.current.stop();+ return;+ }+ if (status !== "idle") return;++ try {+ const Recognition = getSpeechRecognition();+ if (endpoint.baseUrl) {+ await startRecording(endpoint);+ } else if (Recognition) {+ startSpeechRecognition(Recognition);+ }+ } catch (error) {+ setStatus("idle");+ onError(error);+ }+ };++ return { status, isSupported, toggle };+}diff --git a/src/i18n/translation.json b/src/i18n/translation.jsonindex 763713a205..0517f26070 100644--- a/src/i18n/translation.json+++ b/src/i18n/translation.json@@ -43620,5 +43620,90 @@"ca": "La sincronització de Git només està disponible en connectar-se a un backend local d'OpenHands.","tr": "Git Senkronizasyonu yalnızca yerel bir OpenHands arka ucuna bağlıyken kullanılabilir.","uk": "Git-синхронізація доступна лише при підключенні до локального бекенду OpenHands."+ },+ "CHAT_INTERFACE$START_DICTATION": {+ "en": "Start dictation",+ "ja": "音声入力を開始",+ "zh-CN": "开始语音输入",+ "zh-TW": "開始語音輸入",+ "ko-KR": "음성 입력 시작",+ "no": "Start diktering",+ "it": "Avvia dettatura",+ "pt": "Iniciar ditado",+ "es": "Iniciar dictado",+ "ar": "بدء الإملاء الصوتي",+ "fr": "Démarrer la dictée",+ "tr": "Dikteyi başlat",+ "de": "Diktat starten",+ "uk": "Почати диктування",+ "ca": "Inicia el dictat"+ },+ "CHAT_INTERFACE$STOP_DICTATION": {+ "en": "Stop dictation",+ "ja": "音声入力を停止",+ "zh-CN": "停止语音输入",+ "zh-TW": "停止語音輸入",+ "ko-KR": "음성 입력 중지",+ "no": "Stopp diktering",+ "it": "Interrompi dettatura",+ "pt": "Parar ditado",+ "es": "Detener dictado",+ "ar": "إيقاف الإملاء الصوتي",+ "fr": "Arrêter la dictée",+ "tr": "Dikteyi durdur",+ "de": "Diktat beenden",+ "uk": "Зупинити диктування",+ "ca": "Atura el dictat"+ },+ "CHAT_INTERFACE$DICTATION_FAILED": {+ "en": "Dictation failed. Check microphone access and your voice input settings.",+ "ja": "音声入力に失敗しました。マイクへのアクセスと音声入力の設定を確認してください。",+ "zh-CN": "语音输入失败。请检查麦克风权限和语音输入设置。",+ "zh-TW": "語音輸入失敗。請檢查麥克風權限和語音輸入設定。",+ "ko-KR": "음성 입력에 실패했습니다. 마이크 접근 권한과 음성 입력 설정을 확인하세요.",+ "no": "Dikteringen mislyktes. Sjekk mikrofontilgang og innstillingene for taleinndata.",+ "it": "Dettatura non riuscita. Controlla l'accesso al microfono e le impostazioni di input vocale.",+ "pt": "Falha no ditado. Verifique o acesso ao microfone e as configurações de entrada de voz.",+ "es": "El dictado falló. Comprueba el acceso al micrófono y la configuración de entrada de voz.",+ "ar": "فشل الإملاء الصوتي. تحقق من إذن الوصول إلى الميكروفون وإعدادات الإدخال الصوتي.",+ "fr": "La dictée a échoué. Vérifiez l'accès au microphone et vos paramètres de saisie vocale.",+ "tr": "Dikte başarısız oldu. Mikrofon erişimini ve ses girişi ayarlarınızı kontrol edin.",+ "de": "Diktat fehlgeschlagen. Prüfe den Mikrofonzugriff und deine Spracheingabe-Einstellungen.",+ "uk": "Не вдалося виконати диктування. Перевірте доступ до мікрофона та налаштування голосового введення.",+ "ca": "El dictat ha fallat. Comprova l'accés al micròfon i la configuració de l'entrada de veu."+ },+ "SETTINGS$VOICE_INPUT": {+ "en": "Voice input",+ "ja": "音声入力",+ "zh-CN": "语音输入",+ "zh-TW": "語音輸入",+ "ko-KR": "음성 입력",+ "no": "Taleinndata",+ "it": "Input vocale",+ "pt": "Entrada de voz",+ "es": "Entrada de voz",+ "ar": "الإدخال الصوتي",+ "fr": "Saisie vocale",+ "tr": "Ses girişi",+ "de": "Spracheingabe",+ "uk": "Голосове введення",+ "ca": "Entrada de veu"+ },+ "SETTINGS$VOICE_INPUT_DESCRIPTION": {+ "en": "The microphone button in the chat input uses your browser's speech recognition. To use your own transcription service instead, such as a local Whisper server, enter an OpenAI-compatible endpoint. These settings are stored in this browser only.",+ "ja": "チャット入力のマイクボタンはブラウザの音声認識を使用します。ローカルの Whisper サーバーなど独自の文字起こしサービスを使うには、OpenAI 互換のエンドポイントを入力してください。これらの設定はこのブラウザにのみ保存されます。",+ "zh-CN": "聊天输入框中的麦克风按钮使用浏览器的语音识别。若要改用你自己的转写服务(例如本地 Whisper 服务器),请输入兼容 OpenAI 的端点。这些设置仅保存在此浏览器中。",+ "zh-TW": "聊天輸入框中的麥克風按鈕使用瀏覽器的語音辨識。若要改用你自己的轉寫服務(例如本機 Whisper 伺服器),請輸入相容 OpenAI 的端點。這些設定僅儲存在此瀏覽器中。",+ "ko-KR": "채팅 입력의 마이크 버튼은 브라우저의 음성 인식을 사용합니다. 로컬 Whisper 서버 등 자체 음성 변환 서비스를 사용하려면 OpenAI 호환 엔드포인트를 입력하세요. 이 설정은 이 브라우저에만 저장됩니다.",+ "no": "Mikrofonknappen i chatteinndataen bruker nettleserens talegjenkjenning. For å bruke din egen transkripsjonstjeneste i stedet, for eksempel en lokal Whisper-server, angir du et OpenAI-kompatibelt endepunkt. Disse innstillingene lagres bare i denne nettleseren.",+ "it": "Il pulsante del microfono nell'input della chat usa il riconoscimento vocale del browser. Per usare invece un tuo servizio di trascrizione, ad esempio un server Whisper locale, inserisci un endpoint compatibile con OpenAI. Queste impostazioni vengono salvate solo in questo browser.",+ "pt": "O botão de microfone na entrada do chat usa o reconhecimento de voz do navegador. Para usar seu próprio serviço de transcrição, como um servidor Whisper local, informe um endpoint compatível com OpenAI. Essas configurações ficam salvas apenas neste navegador.",+ "es": "El botón de micrófono de la entrada del chat usa el reconocimiento de voz del navegador. Para usar tu propio servicio de transcripción, como un servidor Whisper local, introduce un endpoint compatible con OpenAI. Esta configuración solo se guarda en este navegador.",+ "ar": "يستخدم زر الميكروفون في حقل الدردشة ميزة التعرف على الكلام في متصفحك. لاستخدام خدمة نسخ خاصة بك بدلاً من ذلك، مثل خادم Whisper محلي، أدخل نقطة نهاية متوافقة مع OpenAI. تُحفظ هذه الإعدادات في هذا المتصفح فقط.",+ "fr": "Le bouton micro de la zone de saisie utilise la reconnaissance vocale de votre navigateur. Pour utiliser plutôt votre propre service de transcription, comme un serveur Whisper local, saisissez un endpoint compatible OpenAI. Ces paramètres sont enregistrés uniquement dans ce navigateur.",+ "tr": "Sohbet girişindeki mikrofon düğmesi tarayıcınızın konuşma tanıma özelliğini kullanır. Bunun yerine yerel bir Whisper sunucusu gibi kendi transkripsiyon hizmetinizi kullanmak için OpenAI uyumlu bir uç nokta girin. Bu ayarlar yalnızca bu tarayıcıda saklanır.",+ "de": "Die Mikrofontaste im Chat-Eingabefeld verwendet die Spracherkennung deines Browsers. Um stattdessen deinen eigenen Transkriptionsdienst zu nutzen, etwa einen lokalen Whisper-Server, gib einen OpenAI-kompatiblen Endpunkt ein. Diese Einstellungen werden nur in diesem Browser gespeichert.",+ "uk": "Кнопка мікрофона в полі чату використовує розпізнавання мовлення вашого браузера. Щоб натомість використовувати власний сервіс транскрибування, наприклад локальний сервер Whisper, введіть OpenAI-сумісну кінцеву точку. Ці налаштування зберігаються лише в цьому браузері.",+ "ca": "El botó del micròfon de l'entrada del xat utilitza el reconeixement de veu del navegador. Per utilitzar el teu propi servei de transcripció, com ara un servidor Whisper local, introdueix un endpoint compatible amb OpenAI. Aquesta configuració només es desa en aquest navegador."}}diff --git a/src/routes/app-settings.tsx b/src/routes/app-settings.tsxindex f88618a3d8..6c86e04540 100644--- a/src/routes/app-settings.tsx+++ b/src/routes/app-settings.tsx@@ -12,6 +12,7 @@ import { I18nKey } from "#/i18n/declaration";import { LanguageInput } from "#/components/features/settings/app-settings/language-input";import { ThemeInput } from "#/components/features/settings/app-settings/theme-input";import { GettingStartedChecklistSwitch } from "#/components/features/settings/app-settings/getting-started-checklist-switch";+import { VoiceInputSettings } from "#/components/features/settings/app-settings/voice-input-settings";import {displayErrorToast,displaySuccessToast,@@ -258,6 +259,8 @@ export function AppSettingsScreen() {</NavigationLink></div>+ <VoiceInputSettings />+<div className="border-t border-border pt-6 mt-2"><h3 className="text-lg font-medium mb-2">{t(I18nKey.SETTINGS$GIT_SETTINGS)}diff --git a/src/utils/transcription-endpoint-storage.ts b/src/utils/transcription-endpoint-storage.tsnew file mode 100644index 0000000000..519f9d264d--- /dev/null+++ b/src/utils/transcription-endpoint-storage.ts@@ -0,0 +1,58 @@+export const TRANSCRIPTION_ENDPOINT_STORAGE_KEY =+ "openhands-transcription-endpoint";++/** The API key lives in sessionStorage so it is not persisted at rest. */+export const TRANSCRIPTION_API_KEY_STORAGE_KEY =+ "openhands-transcription-api-key";++export const DEFAULT_TRANSCRIPTION_MODEL = "whisper-1";++/** OpenAI-compatible `/audio/transcriptions` provider, kept in this browser only. */+export interface TranscriptionEndpoint {+ baseUrl: string;+ apiKey: string;+ model: string;+}++const EMPTY_TRANSCRIPTION_ENDPOINT: TranscriptionEndpoint = {+ baseUrl: "",+ apiKey: "",+ model: "",+};++const readString = (value: unknown): string =>+ typeof value === "string" ? value.trim() : "";++export function readTranscriptionEndpoint(): TranscriptionEndpoint {+ if (typeof window === "undefined") return EMPTY_TRANSCRIPTION_ENDPOINT;++ try {+ const stored = JSON.parse(+ window.localStorage.getItem(TRANSCRIPTION_ENDPOINT_STORAGE_KEY) ?? "{}",+ );+ return {+ baseUrl: readString(stored?.baseUrl),+ apiKey: readString(+ window.sessionStorage.getItem(TRANSCRIPTION_API_KEY_STORAGE_KEY),+ ),+ model: readString(stored?.model),+ };+ } catch {+ return EMPTY_TRANSCRIPTION_ENDPOINT;+ }+}++export function writeTranscriptionEndpoint(+ update: Partial<TranscriptionEndpoint>,+): void {+ try {+ const { apiKey, ...rest } = { ...readTranscriptionEndpoint(), ...update };+ window.localStorage.setItem(+ TRANSCRIPTION_ENDPOINT_STORAGE_KEY,+ JSON.stringify(rest),+ );+ window.sessionStorage.setItem(TRANSCRIPTION_API_KEY_STORAGE_KEY, apiKey);+ } catch {+ // Ignore storage failures; dictation falls back to browser recognition.+ }+}