AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(frontend/copilot): when transcription fails, keep the recording and offer retry + download (#14706)
Bentlyd8426457e59b
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/ChatContainer.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/ChatContainer.tsxindex 2a5b218f29..bfde2884ec 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/ChatContainer.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/ChatContainer.tsx@@ -402,6 +402,9 @@ export const ChatContainer = ({<VoiceModeBarstate={voice.state}statusLabel={voice.statusLabel}+ failure={voice.failure}+ onRetry={voice.retryFailedUtterance}+ onDownload={voice.downloadFailedUtterance}leaveButton={<VoiceModeButtonisActivediff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsxindex c3e4627d11..fad4338fad 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsx@@ -35,6 +35,7 @@ import { MentionDropdown } from "./components/MentionDropdown";import { ConnectionPicker } from "./components/ConnectionPicker/ConnectionPicker";import { RecordingButton } from "./components/RecordingButton";import { RecordingIndicator } from "./components/RecordingIndicator";+import { TranscriptionErrorBar } from "./components/TranscriptionErrorBar";import { WorkspaceFilePicker } from "./components/WorkspaceFilePicker/WorkspaceFilePicker";import { useCopilotUIStore } from "../../store";import { isTokenDevtoolEnabled } from "../../tokenDevtool/gate";@@ -205,6 +206,11 @@ export function ChatInput({const {isRecording,isTranscribing,+ transcriptionError,+ hasFailedRecording,+ retryTranscription,+ downloadFailedRecording,+ dismissTranscriptionError,elapsedTime,toggleRecording,handleKeyDown: voiceHandleKeyDown,@@ -328,6 +334,16 @@ export function ChatInput({)}>{voiceBar}+ {!voiceBar && transcriptionError && hasFailedRecording && (+ <TranscriptionErrorBar+ message={transcriptionError}+ isRetrying={isTranscribing}+ onRetry={retryTranscription}+ onDownload={downloadFailedRecording}+ onDismiss={dismissTranscriptionError}+ className={stacked ? undefined : "mt-1.5"}+ />+ )}<FileChipsattachments={attachments}onRemove={handleRemoveAttachment}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/ChatInput.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/ChatInput.test.tsxindex 033c8060a6..6c9ada84bd 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/ChatInput.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/ChatInput.test.tsx@@ -110,10 +110,22 @@ vi.mock("@/components/molecules/Toast/use-toast", () => ({useToast: () => ({ toast: vi.fn(), dismiss: vi.fn() }),}));+const voiceRetry = vi.fn();+const voiceDownload = vi.fn();+const voiceDismiss = vi.fn();+let mockTranscriptionError: string | null = null;+let mockHasFailedRecording = false;+let mockIsTranscribing = false;+vi.mock("../useVoiceRecording", () => ({useVoiceRecording: () => ({isRecording: false,- isTranscribing: false,+ isTranscribing: mockIsTranscribing,+ transcriptionError: mockTranscriptionError,+ hasFailedRecording: mockHasFailedRecording,+ retryTranscription: voiceRetry,+ downloadFailedRecording: voiceDownload,+ dismissTranscriptionError: voiceDismiss,elapsedTime: 0,toggleRecording: vi.fn(),handleKeyDown: vi.fn(),@@ -233,6 +245,9 @@ afterEach(() => {mockFlagValue = false;mockTokenDevtoolEnabled = false;mockInitialPrompt = null;+ mockTranscriptionError = null;+ mockHasFailedRecording = false;+ mockIsTranscribing = false;});describe("ChatInput composer row", () => {@@ -1121,6 +1136,71 @@ describe("ChatInput voice mode", () => {});});+describe("ChatInput transcription failure", () => {+ const mockOnSend = vi.fn();++ it("offers a retry and the recording itself when transcription fails", () => {+ mockTranscriptionError = "Transcription service unavailable";+ mockHasFailedRecording = true;+ render(<ChatInput onSend={mockOnSend} />);++ expect(screen.getByRole("alert").textContent).toContain(+ "Transcription service unavailable",+ );++ fireEvent.click(screen.getByRole("button", { name: /^Retry$/ }));+ fireEvent.click(screen.getByRole("button", { name: /Download recording/ }));+ fireEvent.click(screen.getByRole("button", { name: "Dismiss" }));++ expect(voiceRetry).toHaveBeenCalledTimes(1);+ expect(voiceDownload).toHaveBeenCalledTimes(1);+ expect(voiceDismiss).toHaveBeenCalledTimes(1);+ });++ it("keeps the row up while the retry is in flight", () => {+ // It must not blink away and back: that takes the Retry button with it+ // and makes a slow retry look like it did nothing.+ mockTranscriptionError = "Transcription failed";+ mockHasFailedRecording = true;+ mockIsTranscribing = true;+ render(<ChatInput onSend={mockOnSend} />);++ expect(screen.getByRole("alert")).toBeDefined();+ expect(+ screen.getByRole("button", { name: /^Retry$/ }).hasAttribute("disabled"),+ ).toBe(true);+ });++ it("renders in the card composer too", () => {+ // The empty-state composer stacks its rows and supplies its own spacing.+ mockTranscriptionError = "Transcription failed";+ mockHasFailedRecording = true;+ render(<ChatInput onSend={mockOnSend} stacked />);++ expect(screen.getByRole("alert").textContent).toContain(+ "Transcription failed",+ );+ });++ it("says nothing when there is no failed recording to act on", () => {+ render(<ChatInput onSend={mockOnSend} />);++ expect(screen.queryByRole("alert")).toBeNull();+ });++ it("stays out of voice mode's way", () => {+ // Voice mode replaces the composer's controls; a dictation error from+ // before it was entered must not squat on top of the voice bar.+ mockTranscriptionError = "Transcription failed";+ mockHasFailedRecording = true;+ render(+ <ChatInput onSend={mockOnSend} voiceBar={<div data-testid="bar" />} />,+ );++ expect(screen.queryByRole("alert")).toBeNull();+ });+});+/** `display: none` on an ancestor, which `toBeVisible` cannot see in jsdom. */function isShown(element: HTMLElement): boolean {return !element.closest(".hidden");diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useVoiceRecording.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useVoiceRecording.test.tsxindex df384f8b11..ff72b11173 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useVoiceRecording.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useVoiceRecording.test.tsx@@ -11,8 +11,15 @@ vi.mock("@/components/molecules/Toast/use-toast", () => ({useToast: () => ({ toast: vi.fn() }),}));+const downloaded: Blob[] = [];+vi.mock("../../../voice/downloadRecording", () => ({+ downloadRecording: (blob: Blob) => downloaded.push(blob),+}));+const getUserMedia = vi.fn();const recorderStop = vi.fn();+/** Only the transcription tests want a real stop; the rest assert on the call. */+let firesOnStop = false;function keydown(key: string, isComposing = false) {return {@@ -35,9 +42,19 @@ class FakeMediaRecorder {ondataavailable: ((event: unknown) => void) | null = null;onstop: (() => void) | null = null;start = vi.fn();- stop = recorderStop;+ stop = () => {+ recorderStop();+ if (!firesOnStop) return;+ this.ondataavailable?.({+ data: new Blob([RECORDED], { type: "audio/webm" }),+ });+ this.onstop?.();+ };}+/** Unique, so a re-sent recording is provably the recorded one. */+const RECORDED = "the-one-recording";+async function renderRecording() {getUserMedia.mockResolvedValue({ getTracks: () => [] });const { result } = renderVoice();@@ -50,11 +67,16 @@ async function renderRecording() {return result;}+/** Shared so a test can assert the transcript reached the composer. */+const setValue = vi.fn();+function renderVoice(value = "") {- return renderHook(() => useVoiceRecording({ setValue: vi.fn(), value }));+ return renderHook(() => useVoiceRecording({ setValue, value }));}beforeEach(() => {+ firesOnStop = false;+ downloaded.length = 0;getUserMedia.mockRejectedValue(new Error("no microphone"));vi.stubGlobal("MediaRecorder", FakeMediaRecorder);Object.defineProperty(navigator, "mediaDevices", {@@ -134,3 +156,204 @@ describe("useVoiceRecording Space shortcut", () => {expect(result.current.isRecording).toBe(true);});});++describe("useVoiceRecording transcription failures", () => {+ const fetchMock = vi.fn();++ beforeEach(() => {+ firesOnStop = true;+ fetchMock.mockReset();+ vi.stubGlobal("fetch", fetchMock);+ vi.spyOn(console, "error").mockImplementation(() => undefined);+ });++ it("keeps the recording and offers it back when transcription fails", async () => {+ failTranscription("Transcription service unavailable");+ const result = await recordAndStop();++ await waitFor(() =>+ expect(result.current.transcriptionError).toBe(+ "Transcription service unavailable",+ ),+ );+ // The whole point: the audio survives the failure.+ expect(result.current.hasFailedRecording).toBe(true);++ act(() => result.current.downloadFailedRecording());+ expect(downloaded).toHaveLength(1);+ expect(await downloaded[0].text()).toBe(RECORDED);+ });++ it("re-sends the same recording on retry, without recording again", async () => {+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ fetchMock.mockResolvedValue({+ ok: true,+ json: async () => ({ text: "the words I said" }),+ });+ await act(async () => result.current.retryTranscription());++ expect(fetchMock).toHaveBeenCalledTimes(2);+ expect(await sentAudio(0)).toBe(RECORDED);+ expect(await sentAudio(1)).toBe(RECORDED);+ // One microphone session: the retry reused the audio, it did not re-record.+ expect(getUserMedia).toHaveBeenCalledTimes(1);+ await waitFor(() => expect(result.current.transcriptionError).toBeNull());+ expect(result.current.hasFailedRecording).toBe(false);+ });++ it("still has the recording after a retry fails as well", async () => {+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ await act(async () => result.current.retryTranscription());++ expect(fetchMock).toHaveBeenCalledTimes(2);+ expect(result.current.transcriptionError).toBe("Transcription failed");+ expect(result.current.hasFailedRecording).toBe(true);+ });++ it("does not lose the recording to a denied microphone prompt", async () => {+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ getUserMedia.mockRejectedValue(+ new DOMException("denied", "NotAllowedError"),+ );+ await act(async () => result.current.startRecording());++ expect(result.current.hasFailedRecording).toBe(true);+ expect(result.current.transcriptionError).toBe("Transcription failed");+ });++ it("does not lose the recording when the recorder itself refuses", async () => {+ // getUserMedia can resolve and the MediaRecorder still throw — an+ // unsupported mime type, a track that died. The catch has no way to give+ // the previous recording back, so it must not have been cleared yet.+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ getUserMedia.mockResolvedValue({ getTracks: () => [] });+ vi.stubGlobal(+ "MediaRecorder",+ class {+ static isTypeSupported = () => false;+ constructor() {+ throw new DOMException("mime not supported", "NotSupportedError");+ }+ },+ );+ await act(async () => result.current.startRecording());++ expect(result.current.isRecording).toBe(false);+ expect(result.current.hasFailedRecording).toBe(true);+ expect(result.current.transcriptionError).toBe("Transcription failed");+ });++ it("supersedes the failed recording once a new one is under way", async () => {+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ getUserMedia.mockResolvedValue({ getTracks: () => [] });+ await act(async () => result.current.startRecording());++ expect(result.current.hasFailedRecording).toBe(false);+ expect(result.current.transcriptionError).toBeNull();+ });++ it("falls back to a readable message when the failure is not an Error", async () => {+ // fetch itself rejecting — offline, DNS — does not always throw an Error,+ // and "undefined" is not a message anyone can act on.+ fetchMock.mockRejectedValue("network is down");+ const result = await recordAndStop();++ await waitFor(() =>+ expect(result.current.transcriptionError).toBe("Transcription failed"),+ );+ expect(result.current.hasFailedRecording).toBe(true);+ });++ it("does nothing when there is no recording to retry", async () => {+ const { result } = renderVoice();++ act(() => result.current.retryTranscription());++ expect(fetchMock).not.toHaveBeenCalled();+ });++ it("drops the failed recording once the user dismisses it", async () => {+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ act(() => result.current.dismissTranscriptionError());++ expect(result.current.transcriptionError).toBeNull();+ expect(result.current.hasFailedRecording).toBe(false);+ act(() => result.current.downloadFailedRecording());+ expect(downloaded).toHaveLength(0);+ });++ it("stays dismissed when the retry it was dismissed during then fails", async () => {+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ let failRetry!: (error: Error) => void;+ fetchMock.mockReturnValue(new Promise((_, reject) => (failRetry = reject)));+ await act(async () => result.current.retryTranscription());+ act(() => result.current.dismissTranscriptionError());+ expect(result.current.transcriptionError).toBeNull();++ await act(async () => {+ failRetry(new Error("Transcription failed"));+ });++ // The row the user waved away does not come back a second later.+ expect(result.current.transcriptionError).toBeNull();+ expect(result.current.hasFailedRecording).toBe(false);+ expect(result.current.isTranscribing).toBe(false);+ });++ it("still delivers a transcript that lands after a dismissal", async () => {+ // Dismiss means "stop showing me this", not "throw away what I said".+ failTranscription("Transcription failed");+ const result = await recordAndStop();+ await waitFor(() => expect(result.current.hasFailedRecording).toBe(true));++ let finishRetry!: (value: unknown) => void;+ fetchMock.mockReturnValue(+ new Promise((resolve) => (finishRetry = resolve)),+ );+ await act(async () => result.current.retryTranscription());+ act(() => result.current.dismissTranscriptionError());++ await act(async () => {+ finishRetry({ ok: true, json: async () => ({ text: "what I said" }) });+ });++ expect(setValue).toHaveBeenCalled();+ expect(result.current.transcriptionError).toBeNull();+ });++ function failTranscription(error: string) {+ fetchMock.mockResolvedValue({ ok: false, json: async () => ({ error }) });+ }++ async function recordAndStop() {+ const result = await renderRecording();+ await act(async () => result.current.stopRecording());+ return result;+ }++ async function sentAudio(call: number) {+ const body = fetchMock.mock.calls[call][1].body as FormData;+ return (body.get("audio") as Blob).text();+ }+});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/TranscriptionErrorBar.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/TranscriptionErrorBar.tsxnew file mode 100644index 0000000000..d2cb1ddeb3--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/TranscriptionErrorBar.tsx@@ -0,0 +1,76 @@+"use client";++import { Button } from "@/components/atoms/Button/Button";+import { Icon } from "@/components/atoms/Icon/Icon";+import { cn } from "@/lib/utils";+import {+ Alert02Icon,+ Cancel01Icon,+ Download04Icon,+ RefreshIcon,+} from "@hugeicons/core-free-icons";++interface Props {+ message: string;+ isRetrying: boolean;+ onRetry: () => void;+ onDownload: () => void;+ onDismiss: () => void;+ className?: string;+}++/**+ * The recording behind this message is still in memory, so the row stays put+ * until the user acts on it — retried, downloaded, or dismissed. Anything+ * that expires on its own takes the audio with it.+ */+export function TranscriptionErrorBar({+ message,+ isRetrying,+ onRetry,+ onDownload,+ onDismiss,+ className,+}: Props) {+ return (+ <div+ role="alert"+ className={cn(+ "flex w-full flex-wrap items-center gap-1.5 rounded-2xl bg-red-50 px-3 py-1.5 text-sm text-red-700",+ className,+ )}+ >+ <Icon icon={Alert02Icon} className="h-4 w-4 shrink-0" aria-hidden />+ <span className="min-w-0 flex-1 truncate">{message}</span>+ <Button+ type="button"+ variant="ghost"+ size="xs"+ leadingIcon={RefreshIcon}+ loading={isRetrying}+ disabled={isRetrying}+ onClick={onRetry}+ >+ Retry+ </Button>+ <Button+ type="button"+ variant="ghost"+ size="xs"+ leadingIcon={Download04Icon}+ onClick={onDownload}+ >+ Download recording+ </Button>+ <Button+ type="button"+ variant="icon"+ size="icon-xs"+ aria-label="Dismiss"+ onClick={onDismiss}+ >+ <Icon icon={Cancel01Icon} className="h-3.5 w-3.5" />+ </Button>+ </div>+ );+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.tsindex cb9c28357f..20c903095d 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.ts@@ -8,6 +8,7 @@ import React, {useState,} from "react";import { isKey } from "@/lib/keyboard";+import { downloadRecording } from "../../voice/downloadRecording";const MAX_RECORDING_DURATION = 2 * 60 * 1000; // 2 minutes in ms@@ -29,6 +30,17 @@ export function useVoiceRecording({const [isRecording, setIsRecording] = useState(false);const [isTranscribing, setIsTranscribing] = useState(false);const [error, setError] = useState<string | null>(null);+ // Kept apart from `error`: a failed transcription is answered inline, next+ // to the audio it still holds, not by a toast that expires in five seconds.+ const [transcriptionError, setTranscriptionError] = useState<string | null>(+ null,+ );+ // Up to two minutes of speech. Dropping it on a transient 500 is the whole+ // bug — the user cannot get those two minutes back.+ const [failedRecording, setFailedRecording] = useState<Blob | null>(null);+ // Bumped when the user dismisses. An attempt they have already waved away+ // must not put the row back on screen when it finally fails.+ const attemptRef = useRef(0);const [elapsedTime, setElapsedTime] = useState(0);const mediaRecorderRef = useRef<MediaRecorder | null>(null);@@ -89,8 +101,11 @@ export function useVoiceRecording({const transcribeAudio = useCallback(async (audioBlob: Blob) => {+ const attempt = attemptRef.current;setIsTranscribing(true);setError(null);+ // The previous failure stays on screen through a retry: clearing it here+ // would blink the row away and back, and take the Retry button with it.try {const formData = new FormData();@@ -114,11 +129,21 @@ export function useVoiceRecording({if (data.text) {handleTranscription(data.text);}+ setTranscriptionError(null);+ setFailedRecording(null);} catch (err) {- const message =- err instanceof Error ? err.message : "Transcription failed";- setError(message);console.error("Transcription error:", err);+ // Dismissed while this was in flight: the user is done with this+ // recording, so the failure has nobody to tell. The success path is+ // deliberately not gated the same way — if the words do arrive, the+ // user gets what they dictated rather than losing it twice.+ if (attempt !== attemptRef.current) return;+ const message =+ err instanceof Error && err.message+ ? err.message+ : "Transcription failed";+ setTranscriptionError(message);+ setFailedRecording(audioBlob);} finally {setIsTranscribing(false);}@@ -126,6 +151,22 @@ export function useVoiceRecording({[handleTranscription, inputId],);+ /** Re-sends the recording that failed, byte for byte. */+ function retryTranscription() {+ if (!failedRecording || isTranscribing || isRecordingRef.current) return;+ void transcribeAudio(failedRecording);+ }++ function downloadFailedRecording() {+ if (failedRecording) downloadRecording(failedRecording);+ }++ function dismissTranscriptionError() {+ attemptRef.current += 1;+ setTranscriptionError(null);+ setFailedRecording(null);+ }+const stopRecording = useCallback(() => {if (mediaRecorderRef.current && isRecordingRef.current) {mediaRecorderRef.current.stop();@@ -176,6 +217,13 @@ export function useVoiceRecording({};mediaRecorder.start(1000); // Collect data every second++ // Only once a new recording is genuinely under way. Everything above+ // can still throw — a denied prompt, an unsupported mime type — and the+ // catch has no way to give the previous recording back.+ setTranscriptionError(null);+ setFailedRecording(null);+isRecordingRef.current = true;setIsRecording(true);startTimeRef.current = Date.now();@@ -275,6 +323,11 @@ export function useVoiceRecording({isRecording,isTranscribing,error,+ transcriptionError,+ hasFailedRecording: failedRecording !== null,+ retryTranscription,+ downloadFailedRecording,+ dismissTranscriptionError,elapsedTime,startRecording,stopRecording,diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/VoiceModeBar.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/VoiceModeBar.test.tsxindex 1d07c156aa..7f1cd4e239 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/VoiceModeBar.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/VoiceModeBar.test.tsx@@ -1,7 +1,7 @@import { act } from "@testing-library/react";import { describe, expect, it, vi } from "vitest";-import { render, screen } from "@/tests/integrations/test-utils";+import { fireEvent, render, screen } from "@/tests/integrations/test-utils";vi.mock("../speechLevel", () => ({ readSpeechLevel: vi.fn(() => null) }));@@ -30,6 +30,35 @@ describe("VoiceModeBar", () => {expect(status.getAttribute("aria-live")).toBe("polite");});+ it("offers the retry and the recording when transcription fails", () => {+ const onRetry = vi.fn();+ const onDownload = vi.fn();+ render(+ <VoiceModeBar+ state="listening"+ statusLabel="Listening"+ failure={{ message: "Transcription failed" }}+ onRetry={onRetry}+ onDownload={onDownload}+ leaveButton={<VoiceModeButton isActive onClick={vi.fn()} />}+ />,+ );++ expect(screen.getByRole("alert").textContent).toBe("Transcription failed");+ // The trace says "all well"; it has no business sitting next to an error.+ expect(screen.queryByRole("status")).toBeNull();+ // Leaving voice mode stays reachable with the error up.+ expect(+ screen.getByRole("button", { name: "Leave voice mode" }),+ ).not.toBeNull();++ fireEvent.click(screen.getByRole("button", { name: "Retry" }));+ fireEvent.click(screen.getByRole("button", { name: "Download recording" }));++ expect(onRetry).toHaveBeenCalledTimes(1);+ expect(onDownload).toHaveBeenCalledTimes(1);+ });+it("gives listening, thinking and speaking each their own colour", () => {// The three read as one continuous animation without it: the handover// from the user's turn to Otto's was the part that did not land.diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/downloadRecording.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/downloadRecording.test.tsnew file mode 100644index 0000000000..10184c59cc--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/downloadRecording.test.ts@@ -0,0 +1,82 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import {+ downloadRecording,+ recordingExtension,+ recordingFileName,+} from "../downloadRecording";++describe("recordingExtension", () => {+ it("maps the types MediaRecorder and the VAD actually produce", () => {+ expect(recordingExtension(new Blob([], { type: "audio/webm" }))).toBe(+ "webm",+ );+ expect(recordingExtension(new Blob([], { type: "audio/mp4" }))).toBe("m4a");+ expect(recordingExtension(new Blob([], { type: "audio/wav" }))).toBe("wav");+ });++ it("ignores the codec parameter MediaRecorder appends", () => {+ const blob = new Blob([], { type: "audio/webm;codecs=opus" });++ expect(recordingExtension(blob)).toBe("webm");+ });++ it("falls back rather than producing an extensionless file", () => {+ expect(recordingExtension(new Blob(["x"]))).toBe("webm");+ });+});++describe("recordingFileName", () => {+ it("stamps the name with a time, and no characters a filesystem refuses", () => {+ const name = recordingFileName(+ new Blob([], { type: "audio/wav" }),+ new Date(Date.UTC(2026, 8, 15, 9, 41, 7)),+ );++ expect(name).toBe("voice-recording-2026-09-15-09-41-07.wav");+ });+});++describe("downloadRecording", () => {+ const createObjectURL = vi.fn(() => "blob:recording");+ const revokeObjectURL = vi.fn();++ beforeEach(() => {+ vi.useFakeTimers();+ global.URL.createObjectURL = createObjectURL;+ global.URL.revokeObjectURL = revokeObjectURL;+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.clearAllMocks();+ });++ it("clicks a download link for the blob and cleans the URL up after", () => {+ const clicked: HTMLAnchorElement[] = [];+ const click = vi+ .spyOn(HTMLAnchorElement.prototype, "click")+ .mockImplementation(function (this: HTMLAnchorElement) {+ clicked.push(this);+ });+ const blob = new Blob(["audio"], { type: "audio/webm" });++ downloadRecording(blob, new Date(Date.UTC(2026, 8, 15, 9, 41, 7)));++ expect(createObjectURL).toHaveBeenCalledWith(blob);+ expect(clicked).toHaveLength(1);+ expect(clicked[0].getAttribute("download")).toBe(+ "voice-recording-2026-09-15-09-41-07.webm",+ );+ expect(clicked[0].getAttribute("href")).toBe("blob:recording");+ // Removed from the document straight away, so a second failure does not+ // leave a row of invisible links behind.+ expect(document.querySelector("a[download]")).toBeNull();++ expect(revokeObjectURL).not.toHaveBeenCalled();+ vi.runAllTimers();+ expect(revokeObjectURL).toHaveBeenCalledWith("blob:recording");++ click.mockRestore();+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/micStateMachine.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/micStateMachine.test.tsindex e4fa597107..705f70df84 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/micStateMachine.test.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/micStateMachine.test.ts@@ -74,6 +74,31 @@ describe("voiceReduce", () => {);});+ it("puts a retry back into transcribing from an idle mic", () => {+ expect(+ run(["ENABLE", "SPEECH_START", "SPEECH_END", "TRANSCRIPT_DROPPED"]),+ ).toBe("listening");+ expect(+ run([+ "ENABLE",+ "SPEECH_START",+ "SPEECH_END",+ "TRANSCRIPT_DROPPED",+ "RETRY",+ ]),+ ).toBe("transcribing");+ });++ it("refuses a retry that would race the user's next utterance", () => {+ // Mid-speech, mid-reply or off: the retry has to lose, or two turns+ // arrive for one reply.+ expect(run(["ENABLE", "SPEECH_START", "RETRY"])).toBe("hearing");+ expect(+ run(["ENABLE", "SPEECH_START", "SPEECH_END", "TRANSCRIPT_SENT", "RETRY"]),+ ).toBe("thinking");+ expect(run(["RETRY"])).toBe("off");+ });+it("cannot skip states out of order", () => {expect(run(["ENABLE", "SPEECH_END"])).toBe("listening");expect(run(["ENABLE", "TRANSCRIPT_SENT"])).toBe("listening");diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/useVoiceMode.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/useVoiceMode.test.tsxindex 4fbdbbdec5..4a746dd1ce 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/useVoiceMode.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/__tests__/useVoiceMode.test.tsx@@ -32,14 +32,24 @@ vi.mock("../vadSession", () => ({const spoken: string[] = [];let transcript = "Build me a Slack agent";-let transcribe: () => Promise<string> = async () => transcript;+let transcribe: (audio: Blob) => Promise<string> = async () => transcript;+/** Every blob handed to the transcriber, so a retry can be proved identical. */+const transcribed: Blob[] = [];vi.mock("../speechApi", () => ({synthesizeSpeech: vi.fn(async (text: string) => {spoken.push(text);return new Blob([text]);}),- transcribeUtterance: vi.fn(() => transcribe()),+ transcribeUtterance: vi.fn((audio: Blob) => {+ transcribed.push(audio);+ return transcribe(audio);+ }),+}));++const downloaded: Blob[] = [];+vi.mock("../downloadRecording", () => ({+ downloadRecording: (blob: Blob) => downloaded.push(blob),}));const clicks: string[] = [];@@ -89,6 +99,8 @@ describe("useVoiceMode", () => {});sessions.length = 0;vadLoad = Promise.resolve();+ transcribed.length = 0;+ downloaded.length = 0;transcribe = async () => transcript;transcript = "Build me a Slack agent";vi.spyOn(window.HTMLMediaElement.prototype, "play").mockImplementation(@@ -653,6 +665,174 @@ describe("useVoiceMode", () => {vi.useRealTimers();});+ it("keeps the recording when transcription fails", async () => {+ // The whole bug: a transient 500 used to drop the audio on the floor and+ // leave the user with nothing but a toast.+ const onSend = vi.fn();+ transcribe = async () => {+ throw new Error("Transcription failed");+ };+ const view = render({ onSend });++ await enable(view);+ await speak();++ expect(onSend).not.toHaveBeenCalled();+ expect(view.result.current.failure).toEqual({+ message: "Transcription failed",+ });+ // The mic comes back, as before — the failure is offered, not forced.+ expect(view.result.current.state).toBe("listening");+ });++ it("falls back to a readable message when the failure is not an Error", async () => {+ transcribe = async () => {+ throw "the network went away";+ };+ const view = render({});++ await enable(view);+ await speak();++ expect(view.result.current.failure).toEqual({+ message: "Transcription failed",+ });+ });++ it("retries the same recording, byte for byte, and finishes the turn", async () => {+ const onSend = vi.fn();+ transcribe = async () => {+ throw new Error("Transcription failed");+ };+ const view = render({ onSend });++ await enable(view);+ await speak();+ expect(transcribed).toHaveLength(1);++ transcribe = async () => transcript;+ await act(async () => view.result.current.retryFailedUtterance());++ expect(transcribed).toHaveLength(2);+ expect(transcribed[1]).toBe(transcribed[0]);+ await waitFor(() => expect(onSend).toHaveBeenCalledWith(transcript));+ expect(view.result.current.failure).toBeNull();+ expect(view.result.current.state).toBe("thinking");+ });++ it("still has the recording after a retry fails too", async () => {+ transcribe = async () => {+ throw new Error("Transcription failed");+ };+ const view = render({});++ await enable(view);+ await speak();+ await act(async () => view.result.current.retryFailedUtterance());++ expect(transcribed).toHaveLength(2);+ expect(view.result.current.failure).not.toBeNull();+ expect(view.result.current.state).toBe("listening");++ await act(async () => view.result.current.downloadFailedUtterance());+ expect(downloaded).toEqual([transcribed[0]]);+ });++ it("does not close the session out from under the error", async () => {+ // The mic closes after 8s of silence. Reading an error and deciding takes+ // longer than that, and closing takes the recording with it.+ vi.useFakeTimers();+ transcribe = async () => {+ throw new Error("Transcription failed");+ };+ const view = render({ silenceTimeoutMs: 8_000 });++ await enable(view);+ await speak();+ await act(async () => {+ await vi.advanceTimersByTimeAsync(30_000);+ });++ expect(view.result.current.state).toBe("listening");+ expect(view.result.current.failure).not.toBeNull();++ // It is held open, not held open forever: the session cap still ends it.+ await act(async () => {+ await vi.advanceTimersByTimeAsync(5 * 60 * 1000);+ });+ expect(view.result.current.state).toBe("off");+ vi.useRealTimers();+ });++ it("drops a failure that arrives after the user left voice mode", async () => {+ // Stop while the transcript is in flight, then let it reject. Writing the+ // failure back from a dead turn used to resurrect the old audio under the+ // next activation, with a Retry that would send it.+ let failTranscribing!: (error: Error) => void;+ transcribe = () => new Promise((_, reject) => (failTranscribing = reject));+ const view = render({});++ await enable(view);+ await act(async () => vad.onSpeechStart());+ await act(async () => vad.onSpeechEnd(new Blob(["wav"])));+ expect(view.result.current.state).toBe("transcribing");++ await act(async () => view.result.current.toggle());+ expect(view.result.current.state).toBe("off");+ await act(async () => failTranscribing(new Error("Transcription failed")));+ await act(async () => undefined);++ expect(view.result.current.failure).toBeNull();++ // Restarting gets a clean session, not the last one's error and audio.+ await enable(view);+ expect(view.result.current.state).toBe("listening");+ expect(view.result.current.failure).toBeNull();+ await act(async () => view.result.current.downloadFailedUtterance());+ expect(downloaded).toHaveLength(0);++ // The dead turn is not counted against the new session either.+ expect(+ tracked.filter(([event]) => event === "voice_turn_dropped"),+ ).toHaveLength(0);+ });++ it("forgets the failed recording once the user speaks again", async () => {+ // Otherwise the stale row sits over a live mic, and Retry sends audio the+ // user has already moved on from.+ transcribe = async () => {+ throw new Error("Transcription failed");+ };+ const view = render({});++ await enable(view);+ await speak();+ expect(view.result.current.failure).not.toBeNull();++ await act(async () => vad.onSpeechStart());++ expect(view.result.current.failure).toBeNull();+ await act(async () => view.result.current.downloadFailedUtterance());+ expect(downloaded).toHaveLength(0);+ });++ it("does not retry over an utterance already in flight", async () => {+ transcribe = async () => {+ throw new Error("Transcription failed");+ };+ const onSend = vi.fn();+ const view = render({ onSend });++ await enable(view);+ await speak();+ await act(async () => vad.onSpeechStart());++ await act(async () => view.result.current.retryFailedUtterance());++ expect(transcribed).toHaveLength(1);+ expect(view.result.current.state).toBe("hearing");+ });+it("shuts itself down when the flag goes off", async () => {const view = render({});await enable(view);diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/components/VoiceModeBar.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/components/VoiceModeBar.tsxindex 64ee9722dd..8c9fded79c 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/components/VoiceModeBar.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/components/VoiceModeBar.tsx@@ -1,6 +1,9 @@"use client";+import { Button } from "@/components/atoms/Button/Button";+import type { VoiceState } from "../micStateMachine";+import type { VoiceFailure } from "../useVoiceMode";import { VoiceTrace, type TraceSource } from "./VoiceTrace";/**@@ -26,10 +29,44 @@ interface Props {statusLabel: string;/** Leaves voice mode — and while Otto speaks, is the stop control. */leaveButton?: React.ReactNode;+ /** A transcription that failed with the audio still in hand. */+ failure?: VoiceFailure | null;+ onRetry?: () => void;+ onDownload?: () => void;}-export function VoiceModeBar({ state, statusLabel, leaveButton }: Props) {+export function VoiceModeBar({+ state,+ statusLabel,+ leaveButton,+ failure = null,+ onRetry,+ onDownload,+}: Props) {if (state === "off") return null;++ // The trace says "everything is fine"; a failure has to take its place+ // rather than sit beside it, and it stays until the user acts on it.+ if (failure) {+ return (+ <div className="flex w-full flex-wrap items-center gap-2 py-1.5 pl-3 pr-1.5">+ <span+ role="alert"+ className="min-w-0 flex-1 truncate text-sm text-red-600"+ >+ {failure.message}+ </span>+ <Button type="button" variant="ghost" size="xs" onClick={onRetry}>+ Retry+ </Button>+ <Button type="button" variant="ghost" size="xs" onClick={onDownload}>+ Download recording+ </Button>+ {leaveButton}+ </div>+ );+ }+const { source, color } = APPEARANCE[state];return (diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/downloadRecording.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/downloadRecording.tsnew file mode 100644index 0000000000..ba6bf7e4ea--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/downloadRecording.ts@@ -0,0 +1,42 @@+/**+ * Handing the recording back is the last resort when transcription keeps+ * failing: the user has already said the thing, and re-saying it is the one+ * cost a retry cannot refund.+ */++const EXTENSIONS: Record<string, string> = {+ "audio/webm": "webm",+ "audio/ogg": "ogg",+ "audio/mp4": "m4a",+ "audio/mpeg": "mp3",+ "audio/wav": "wav",+ "audio/x-wav": "wav",+ "audio/wave": "wav",+};++/** `audio/webm;codecs=opus` is what MediaRecorder actually reports. */+function baseType(blob: Blob): string {+ return blob.type.split(";")[0].trim().toLowerCase();+}++export function recordingExtension(blob: Blob): string {+ return EXTENSIONS[baseType(blob)] ?? "webm";+}++export function recordingFileName(blob: Blob, at: Date = new Date()): string {+ const stamp = at.toISOString().slice(0, 19).replace(/[:T]/g, "-");+ return `voice-recording-${stamp}.${recordingExtension(blob)}`;+}++export function downloadRecording(blob: Blob, at?: Date): void {+ const url = URL.createObjectURL(blob);+ const link = document.createElement("a");+ link.href = url;+ link.download = recordingFileName(blob, at);+ document.body.appendChild(link);+ link.click();+ link.remove();+ // Immediate revocation cancels the download in Safari; one tick is enough+ // for every browser to have taken the blob.+ setTimeout(() => URL.revokeObjectURL(url), 0);+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/micStateMachine.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/micStateMachine.tsindex f24809e7f0..ee98994f3b 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/micStateMachine.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/micStateMachine.ts@@ -24,6 +24,8 @@ export type VoiceEvent =| { type: "TRANSCRIPT_SENT" }/** Transcript was empty, filler, or the send failed. */| { type: "TRANSCRIPT_DROPPED" }+ /** The user asked for a failed transcription to be tried again. */+ | { type: "RETRY" }| { type: "REPLY_SPEAKING" }/** The reply finished with nothing left to say. */| { type: "REPLY_DONE" }@@ -44,6 +46,10 @@ export function voiceReduce(state: VoiceState, event: VoiceEvent): VoiceState {return state === "transcribing" ? "thinking" : state;case "TRANSCRIPT_DROPPED":return state === "transcribing" ? "listening" : state;+ // Only from an open, quiet mic: mid-utterance the retry would race the+ // user's new speech for the same turn.+ case "RETRY":+ return state === "listening" ? "transcribing" : state;case "REPLY_SPEAKING":return state === "thinking" || state === "speaking" ? "speaking" : state;case "REPLY_DONE":diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/useVoiceMode.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/useVoiceMode.tsindex c7494d6826..427bd2a90b 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/voice/useVoiceMode.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/voice/useVoiceMode.ts@@ -8,6 +8,7 @@ import { trackVoiceMode } from "@/services/copilot/voice-mode-analytics";import { primeAudioContext } from "./audioContext";import { playClickSound } from "./clickSound";+import { downloadRecording } from "./downloadRecording";import {describeVoiceState,isMicOpen,@@ -42,6 +43,11 @@ const REPLY_SILENCE_MS = 90_000;*/const TEXT_SETTLED_MS = 800;+/** A transcription that failed with its audio still held. */+export interface VoiceFailure {+ message: string;+}+interface Args {enabled: boolean;messages: UIMessage[];@@ -97,6 +103,10 @@ export function useVoiceMode({// value, which is exactly the double-click that used to leak a session.const starting = useRef(false);const [isStarting, setIsStarting] = useState(false);+ // A transcription that failed must not take the audio with it: the user has+ // already spoken, and nothing else on this page can give that back.+ const failedUtterance = useRef<Blob | null>(null);+ const [failure, setFailure] = useState<VoiceFailure | null>(null);useEffect(() => {if (stateRef.current === "thinking" || stateRef.current === "speaking") {@@ -133,6 +143,9 @@ export function useVoiceMode({isStarting,statusLabel: describeVoiceState(state),toggle,+ failure,+ retryFailedUtterance,+ downloadFailedUtterance,};function toggle() {@@ -150,7 +163,12 @@ export function useVoiceMode({let session: VadSession;try {session = await startVadSession({- onSpeechStart: () => dispatch({ type: "SPEECH_START" }),+ onSpeechStart: () => {+ // A new utterance supersedes the failed one; leaving the old error+ // on screen while the mic is live invites a retry of the wrong audio.+ clearFailure();+ dispatch({ type: "SPEECH_START" });+ },onMisfire: () => {trackVoiceMode("voice_turn_dropped", { reason: "vad_misfire" });dispatch({ type: "SPEECH_MISFIRE" });@@ -202,6 +220,7 @@ export function useVoiceMode({activation.current += 1;setVoiceTurnActive(false);setStarting(false);+ clearFailure();clearTimers();playerRef.current?.stop();discardReply();@@ -222,16 +241,71 @@ export function useVoiceMode({}async function handleUtterance(wav: Blob) {- const mine = activation.current;utteranceEndedAt.current = Date.now();dispatch({ type: "SPEECH_END" });playClickSound();+ await transcribeAndSend(wav);+ }++ /** Retries the utterance that failed, byte for byte. */+ async function retryFailedUtterance() {+ const wav = failedUtterance.current;+ if (!wav) return;+ dispatch({ type: "RETRY" });+ // Refused — the user is already speaking again, or the session has moved+ // on. The failure stays up, audio and all; dropping it here would lose+ // the recording to a click that did nothing.+ if (stateRef.current !== "transcribing") return;+ clearFailure();+ trackVoiceMode("voice_transcribe_retried", {+ turn_index: turnIndex.current,+ });+ utteranceEndedAt.current = Date.now();+ await transcribeAndSend(wav);+ }++ function downloadFailedUtterance() {+ const wav = failedUtterance.current;+ if (!wav) return;+ trackVoiceMode("voice_recording_downloaded", {+ turn_index: turnIndex.current,+ });+ downloadRecording(wav);+ }++ function clearFailure() {+ failedUtterance.current = null;+ setFailure(null);+ }++ /** Assumes the state is already `transcribing`. */+ async function transcribeAndSend(wav: Blob) {+ const mine = activation.current;let transcript = "";try {transcript = await transcribeUtterance(wav);} catch (error) {- report(error);+ // A session the user has already left owns none of this: writing the+ // failure back here would resurrect the old recording under the next+ // activation, and re-arm a session timer deactivate had just cleared.+ if (mine !== activation.current || stateRef.current !== "transcribing") {+ return;+ }+ console.error("[Voice mode]", error);+ // No toast: it takes the only retry affordance off screen with it after+ // five seconds, and the audio behind it is unrecoverable once dropped.+ failedUtterance.current = wav;+ setFailure({+ message:+ error instanceof Error && error.message+ ? error.message+ : "Transcription failed",+ });+ // The mic comes back, but its 8-second silence close must not: that+ // would take the recording with it before the user has read the error.+ // The session cap is what ends an abandoned session from here.+ setTimer("session", MAX_SESSION_MS, () => deactivate("silence_timeout"));trackVoiceMode("voice_turn_dropped", { reason: "transcribe_failed" });dispatch({ type: "TRANSCRIPT_DROPPED" });return;@@ -375,7 +449,7 @@ export function useVoiceMode({if (isMicOpen(next)) vadRef.current?.resume();else vadRef.current?.pause();- if (next === "listening") {+ if (next === "listening" && !failedUtterance.current) {setTimer("silence", inputs.current.silenceTimeoutMs, () =>deactivate("silence_timeout"),);diff --git a/autogpt_platform/frontend/src/services/copilot/voice-mode-analytics.ts b/autogpt_platform/frontend/src/services/copilot/voice-mode-analytics.tsindex 35dcc02ee6..78d23b69b7 100644--- a/autogpt_platform/frontend/src/services/copilot/voice-mode-analytics.ts+++ b/autogpt_platform/frontend/src/services/copilot/voice-mode-analytics.ts@@ -33,6 +33,13 @@ type VoiceModeEvent =| "voice_first_sound_latency_ms"// The mic reopened after a reply finished playing: a full loop closed.| "voice_turn_completed"+ // The user asked for a failed transcription to be tried again. Against+ // `voice_turn_dropped{reason:transcribe_failed}` this says how many of those+ // dropped turns the user actually got back.+ | "voice_transcribe_retried"+ // Gave up on transcription and took the audio instead. Rare by design: a+ // run of these means retrying is not working.+ | "voice_recording_downloaded"| "voice_mode_permission_denied"// Synthesis or the VAD failed. `stage` says which.| "voice_mode_error";