Skip to content

Open source CapCut alternative.

TypeScript

fix(site): dictation falls back to hosted transcription

David Leeb07deac57e9
diff --git a/site/src/cut/lib/cloudTranscribe.ts b/site/src/cut/lib/cloudTranscribe.ts
index feb6e236..ef96255a 100644
--- a/site/src/cut/lib/cloudTranscribe.ts
+++ b/site/src/cut/lib/cloudTranscribe.ts
@@ -13,6 +13,7 @@
import { renderMix as mixAudio } from "./audioMix";
import { apiFetch } from "./backend";
+import { cloudBackend } from "./backend/cloud";
import { mediaUrl, type SubtitleCue } from "./types";
/** Mirror of the engine's TranscribeSpec (server/transcribe.ts) minus projectId. */
@@ -126,18 +127,24 @@ export class HostedTranscribeError extends Error {
export const isFreeTranscriptionExhausted = (e: unknown): boolean =>
e instanceof HostedTranscribeError && e.code === FREE_EXHAUSTED_CODE;
+/** The transport a run posts its chunks on. Defaults to the active backend;
+ * mic dictation names the hosted one, because the take it is falling back with
+ * is the one the engine could not take. */
+type ChunkTransport = (path: string, init?: RequestInit) => Promise<Response>;
+
async function postChunk(
samples: Float32Array,
offset: number,
locale: string | undefined,
- freeOnly: boolean
+ freeOnly: boolean,
+ request: ChunkTransport
): Promise<WireCue[]> {
const form = new FormData();
form.append("audio", new File([encodeWav(samples)], "chunk.wav", { type: "audio/wav" }));
form.append("offset", String(round(offset)));
if (locale) form.append("locale", locale);
if (freeOnly) form.append("freeOnly", "1");
- const res = await apiFetch("/api/cut/transcribe", { method: "POST", body: form });
+ const res = await request("/api/cut/transcribe", { method: "POST", body: form });
const body = (await res.json().catch(() => null)) as
| { cues?: WireCue[]; error?: string; message?: string }
| null;
@@ -166,7 +173,7 @@ export async function transcribeSamples(
samples: Float32Array,
locale: string | undefined,
isStale?: () => boolean,
- opts?: { freeOnly?: boolean }
+ opts?: { freeOnly?: boolean; request?: ChunkTransport }
): Promise<SubtitleCue[] | null> {
const duration = samples.length / RATE;
const step = (CHUNK_SECONDS - OVERLAP_SECONDS) * RATE;
@@ -188,7 +195,13 @@ export async function transcribeSamples(
const i = next++;
if (i >= chunks.length || failed !== undefined || isStale?.()) return;
try {
- results[i] = await postChunk(chunks[i].slice, chunks[i].offset, locale, opts?.freeOnly ?? false);
+ results[i] = await postChunk(
+ chunks[i].slice,
+ chunks[i].offset,
+ locale,
+ opts?.freeOnly ?? false,
+ opts?.request ?? apiFetch
+ );
} catch (error) {
failed = error;
return;
@@ -231,7 +244,9 @@ export async function transcribeSamples(
}
/** Transcribe a finished mic recording: decode it, downmix/resample to the
- * wire format, run the chunk pipeline, and join the cue texts. */
+ * wire format, run the chunk pipeline, and join the cue texts. It posts to the
+ * hosted route whatever backend the project lives on — the dictation reaches
+ * here when this Mac's engine is not the one transcribing. */
export async function cloudTranscribeRecording(blob: Blob, locale?: string): Promise<string> {
const bytes = await blob.arrayBuffer();
// Decode at the device rate, then resample/downmix through an offline
@@ -252,7 +267,7 @@ export async function cloudTranscribeRecording(blob: Blob, locale?: string): Pro
src.connect(ctx.destination);
src.start();
const mono = (await ctx.startRendering()).getChannelData(0);
- const cues = await transcribeSamples(mono, locale);
+ const cues = await transcribeSamples(mono, locale, undefined, { request: cloudBackend.fetch });
return (cues ?? [])
.map((c) => c.text)
.join(" ")
diff --git a/site/src/cut/lib/micTranscribe.ts b/site/src/cut/lib/micTranscribe.ts
index 35135df2..0abc3227 100644
--- a/site/src/cut/lib/micTranscribe.ts
+++ b/site/src/cut/lib/micTranscribe.ts
@@ -1,27 +1,37 @@
"use client";
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react";
+import { confirmEngine } from "./api";
import { hasLocalCompute } from "./backend";
import { localBackend } from "./backend/local";
import { cloudTranscribeRecording } from "./cloudTranscribe";
// Live dictation for the chat composer. The browser holds the mic permission,
-// so it captures audio here and streams 16 kHz mono s16le PCM to the local
-// engine, which runs it through Apple's on-device SpeechAnalyzer (see
-// server/mic.ts + native/cut-stt.swift). The transcript never leaves the Mac.
-// On the cloud backend there is no engine: the mic records locally
-// (MediaRecorder) and one hosted transcription runs when the user confirms —
-// no live partials, so the composer shows "Listening…" while recording.
+// so capture happens in the page, and the transcription runs wherever it can.
+//
+// Every take is recorded here (MediaRecorder) for the whole dictation — that
+// recording is the floor under the feature. On a Mac with the app, the same
+// audio also streams to the engine as 16 kHz mono s16le PCM for Apple's
+// on-device SpeechAnalyzer (server/mic.ts + native/cut-stt.swift), which gives
+// live partials and keeps the speech on the machine. Where the engine is
+// absent, and the moment it stops carrying a dictation already under way, the
+// recording goes to the hosted route on confirm and the composer shows
+// "Listening…" until it answers. Dictation works on every surface; which
+// transcriber served it is the only difference the user can see.
//
// Capture is an AudioWorklet. It downsamples on the audio thread and hands the
// main thread whole chunks, and it releases its partial chunk on request so the
// end of the last word reaches the model. A dictation that stops carrying audio
-// — a graph that never started, feeds the engine refuses — ends with the reason
-// on screen, because a session the engine never hears from produces nothing and
-// waits for a transcript that will never come.
+// at all — a graph that never started, a microphone delivering nothing — ends
+// with the reason on screen, because a session nothing feeds produces nothing
+// and waits for a transcript that will never come.
export type MicState = "idle" | "starting" | "recording" | "finishing";
+/** Which transcriber is carrying the dictation: this Mac's engine, live, or
+ * the hosted route reading the recording once the user confirms. */
+type MicMode = "engine" | "hosted";
+
const TARGET_RATE = 16000; // must match cut-stt.swift's live input format
const CHUNK_MS = 250; // how much PCM the worklet batches into one message
const FEED_MS = 250; // how often queued PCM is flushed to the engine
@@ -107,6 +117,12 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
const [error, setError] = useState<string | null>(null);
const jobRef = useRef<string | null>(null);
+ /** Which transcriber this dictation is on; a run that loses the engine flips
+ * it to "hosted" without the user doing anything. */
+ const modeRef = useRef<MicMode>("hosted");
+ /** The newest partial, kept outside React so a hosted transcription that
+ * fails can still hand back what the engine had said. */
+ const partialRef = useRef("");
const recorderRef = useRef<MediaRecorder | null>(null);
const recChunksRef = useRef<Blob[]>([]);
const streamRef = useRef<MediaStream | null>(null);
@@ -128,7 +144,7 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
onResultRef.current = onResult;
}, [onResult]);
- /** Stop and forget the cloud recorder (before its tracks stop). */
+ /** Stop and forget the recorded take (before its tracks stop). */
const discardRecorder = useCallback(() => {
const rec = recorderRef.current;
recorderRef.current = null;
@@ -142,7 +158,10 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
}
}, []);
- const teardownAudio = useCallback(() => {
+ /** Drop the live capture that feeds the engine — worklet, context, timers —
+ * and leave the microphone and the recording running. This is what a
+ * dictation sheds when it moves off the engine mid-sentence. */
+ const teardownEngine = useCallback(() => {
if (feedTimer.current) clearInterval(feedTimer.current);
if (pollTimer.current) clearInterval(pollTimer.current);
if (silenceTimer.current) clearTimeout(silenceTimer.current);
@@ -163,31 +182,102 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
ctx.onstatechange = null;
void ctx.close().catch(() => {});
}
- streamRef.current?.getTracks().forEach((t) => t.stop());
- streamRef.current = null;
queueRef.current = [];
feedingRef.current = false;
failuresRef.current = 0;
+ }, []);
+
+ const teardownAudio = useCallback(() => {
+ discardRecorder();
+ teardownEngine();
+ streamRef.current?.getTracks().forEach((t) => t.stop());
+ streamRef.current = null;
gotAudioRef.current = false;
setStream(null);
- }, []);
+ }, [discardRecorder, teardownEngine]);
/** Hand the session back and put the reason on screen. Everything that can
- * leave a dictation unable to carry audio ends here, so the engine never
- * holds a session nothing is feeding and the user is never left watching an
- * empty transcript. */
+ * leave a dictation with no transcriber and no recording ends here, so the
+ * engine never holds a session nothing is feeding and the user is never left
+ * watching an empty transcript. */
const fail = useCallback(
(message: string) => {
const job = jobRef.current;
jobRef.current = null;
if (job) void localBackend.fetch(`/api/cut/mic/${job}/cancel`, { method: "POST" }).catch(() => {});
- discardRecorder();
teardownAudio();
setState("idle");
setPartial("");
+ partialRef.current = "";
setError(message);
},
- [discardRecorder, teardownAudio]
+ [teardownAudio]
+ );
+
+ /** Capture that has delivered nothing by the deadline never will; say so
+ * rather than leaving the user watching an empty transcript. */
+ const armSilence = useCallback(() => {
+ if (silenceTimer.current) clearTimeout(silenceTimer.current);
+ silenceTimer.current = setTimeout(() => {
+ if (gotAudioRef.current) return;
+ fail("No audio is reaching Donkey from this microphone. Check the input device and try again.");
+ }, FIRST_AUDIO_MS);
+ }, [fail]);
+
+ /** Record the take in the page. Every dictation runs this: on the Mac it is
+ * the net under the engine's live transcript, and everywhere else it is the
+ * dictation itself. */
+ const startRecorder = useCallback(
+ (media: MediaStream): boolean => {
+ try {
+ recChunksRef.current = [];
+ const rec = new MediaRecorder(media);
+ rec.ondataavailable = (e) => {
+ if (e.data.size === 0) return;
+ gotAudioRef.current = true;
+ recChunksRef.current.push(e.data);
+ };
+ rec.onerror = () => {
+ recorderRef.current = null;
+ // The engine is still transcribing this one; only a dictation the
+ // recording *was* has nothing left to finish with.
+ if (modeRef.current === "hosted") fail("The browser stopped recording the microphone.");
+ };
+ rec.start(RECORDER_SLICE_MS);
+ recorderRef.current = rec;
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ [fail]
+ );
+
+ /** The engine stopped carrying this dictation. Hand its session back, drop
+ * the live capture, and let the recording finish the job on the hosted route
+ * when the user confirms — the microphone keeps running and nothing on
+ * screen changes but the partial. Returns false when there was no recording
+ * to fall back on, in which case the dictation ended with `reason` shown. */
+ const degrade = useCallback(
+ (reason: string): boolean => {
+ if (!recorderRef.current) {
+ fail(reason);
+ return false;
+ }
+ const job = jobRef.current;
+ jobRef.current = null;
+ if (job) void localBackend.fetch(`/api/cut/mic/${job}/cancel`, { method: "POST" }).catch(() => {});
+ // The app may have quit under the dictation; let the gate settle that
+ // once instead of leaving every other surface to find out on its own.
+ void confirmEngine();
+ modeRef.current = "hosted";
+ teardownEngine();
+ setPartial("");
+ partialRef.current = "";
+ if (!gotAudioRef.current) armSilence();
+ return true;
+ },
+ [fail, teardownEngine, armSilence]
);
/** Concatenate and POST all queued PCM. Guarded so chunks stay ordered. */
@@ -214,15 +304,15 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
failuresRef.current = 0;
} catch (e) {
// A dropped chunk just shortens the transcript; a run of them means the
- // audio is reaching nothing, and the dictation says so.
+ // engine is no longer taking this dictation, and the recording takes over.
failuresRef.current += 1;
if (failuresRef.current >= MAX_FEED_FAILURES) {
- fail(e instanceof Error ? e.message : "Donkey stopped receiving the audio.");
+ degrade(e instanceof Error ? e.message : "Donkey stopped receiving the audio.");
}
} finally {
feedingRef.current = false;
}
- }, [fail]);
+ }, [degrade]);
/** Ask the worklet for the chunk still forming, so the last word is whole. */
const drainTail = useCallback(async () => {
@@ -242,10 +332,101 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
});
}, []);
+ /** Read the evolving transcript off the engine's session. */
+ const pollEngine = useCallback(async () => {
+ const job = jobRef.current;
+ if (!job) return;
+ try {
+ const res = await localBackend.fetch(`/api/cut/mic/${job}`);
+ if (!res.ok) return;
+ const data = (await res.json()) as { text?: string; status?: string; error?: string };
+ if (typeof data.text === "string") {
+ partialRef.current = data.text;
+ setPartial(data.text);
+ }
+ // A session that died — a locale with no model on this Mac, a process
+ // that quit — has no transcript coming; the recording carries it instead.
+ if (data.status === "error") degrade(data.error || "Dictation stopped on this Mac.");
+ } catch {
+ // Transient poll failure; the next tick retries. An engine that is gone
+ // shows up in the feeds, which are what carry the audio.
+ }
+ }, [degrade]);
+
+ /** Open a dictation on this Mac's engine. False when it cannot take one. */
+ const startEngineSession = useCallback(async (): Promise<boolean> => {
+ try {
+ const res = await localBackend.fetch("/api/cut/mic/start", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ locale: navigator.language }),
+ });
+ const data = (await res.json().catch(() => ({}))) as { id?: string };
+ if (!res.ok || !data.id) return false;
+ jobRef.current = data.id;
+ return true;
+ } catch {
+ // The app quit since the gate connected; settle that and go hosted.
+ void confirmEngine();
+ return false;
+ }
+ }, []);
+
+ /** Build the live capture graph that streams PCM to the engine. False when
+ * the dictation ended here, its reason already on screen. */
+ const buildCapture = useCallback(
+ async (media: MediaStream): Promise<boolean> => {
+ try {
+ const ctx = new AudioContext();
+ ctxRef.current = ctx;
+ await ctx.resume();
+ if (ctx.state !== "running") throw new Error("This page is not allowed to capture audio right now.");
+ const url = URL.createObjectURL(new Blob([PCM_WORKLET], { type: "text/javascript" }));
+ try {
+ await ctx.audioWorklet.addModule(url);
+ } finally {
+ URL.revokeObjectURL(url);
+ }
+ const node = new AudioWorkletNode(ctx, "mic-pcm", {
+ numberOfInputs: 1,
+ numberOfOutputs: 1,
+ outputChannelCount: [1],
+ });
+ node.port.onmessage = (e: MessageEvent<{ pcm: Int16Array; last: boolean }>) => {
+ gotAudioRef.current = true;
+ if (e.data.pcm.length > 0) queueRef.current.push(e.data.pcm);
+ if (e.data.last) tailRef.current?.();
+ };
+ // Web Audio pulls the graph from the destination, so the tap reaches it
+ // through a silent gain: the node runs, and nothing echoes the mic.
+ const mute = ctx.createGain();
+ mute.gain.value = 0;
+ const source = ctx.createMediaStreamSource(media);
+ source.connect(node);
+ node.connect(mute);
+ mute.connect(ctx.destination);
+ nodesRef.current = { source, node };
+ // A context that leaves "running" mid-take stops delivering audio and
+ // says nothing; the dictation moves off the engine on the state change
+ // instead of on a transcript that quietly stopped growing.
+ ctx.onstatechange = () => {
+ if (ctxRef.current === ctx && ctx.state !== "running") {
+ degrade("The browser stopped audio capture for this page.");
+ }
+ };
+ return true;
+ } catch (e) {
+ return degrade(e instanceof Error ? e.message : "This browser could not capture the microphone.");
+ }
+ },
+ [degrade]
+ );
+
const start = useCallback(async () => {
if (state !== "idle") return;
setError(null);
setPartial("");
+ partialRef.current = "";
setState("starting");
let media: MediaStream;
try {
@@ -256,172 +437,36 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
return;
}
streamRef.current = media;
+ const recording = startRecorder(media);
// Dictation runs on the Mac whenever the app is there — on-device, free,
- // and live — whatever backend holds the project. Without it the take is
- // recorded and sent to the hosted route in one go, under the same contract:
- // a recorder that cannot start, that dies mid-take, or that hands over
- // nothing ends the dictation with the reason on screen.
- if (!hasLocalCompute()) {
- try {
- recChunksRef.current = [];
- const rec = new MediaRecorder(media);
- rec.ondataavailable = (e) => {
- if (e.data.size === 0) return;
- gotAudioRef.current = true;
- recChunksRef.current.push(e.data);
- };
- rec.onerror = () => fail("The browser stopped recording the microphone.");
- rec.start(RECORDER_SLICE_MS);
- recorderRef.current = rec;
- } catch (e) {
- teardownAudio();
- setState("idle");
- setError(e instanceof Error ? e.message : "This browser could not record the microphone.");
- return;
- }
- setStream(media);
- silenceTimer.current = setTimeout(() => {
- if (gotAudioRef.current) return;
- fail("No audio is reaching Donkey from this microphone. Check the input device and try again.");
- }, FIRST_AUDIO_MS);
- setState("recording");
- return;
- }
- let jobId: string;
- try {
- const res = await localBackend.fetch("/api/cut/mic/start", {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ locale: navigator.language }),
- });
- const data = (await res.json().catch(() => ({}))) as { id?: string; error?: string };
- if (!res.ok || !data.id) throw new Error(data.error || "Dictation needs the Donkey app running on this Mac.");
- jobId = data.id;
- jobRef.current = jobId;
- } catch (e) {
+ // and live — whatever backend holds the project. The result still lands
+ // through the project's own backend, so where the speech ran never decides
+ // where the text goes.
+ const onEngine = hasLocalCompute() && (await startEngineSession());
+ modeRef.current = onEngine ? "engine" : "hosted";
+ // The engine now holds a session, so a capture graph that cannot be built
+ // hands it back rather than leaving a process waiting on audio.
+ if (onEngine && !(await buildCapture(media))) return;
+ if (modeRef.current === "hosted" && !recording) {
teardownAudio();
setState("idle");
- setError(e instanceof Error ? e.message : "Could not start dictation.");
+ setError("This browser could not record the microphone.");
return;
}
-
- // The engine now holds a session, so a capture that cannot be built hands
- // it back here instead of leaving a process waiting on audio.
- try {
- const ctx = new AudioContext();
- ctxRef.current = ctx;
- await ctx.resume();
- if (ctx.state !== "running") throw new Error("This page is not allowed to capture audio right now.");
- const url = URL.createObjectURL(new Blob([PCM_WORKLET], { type: "text/javascript" }));
- try {
- await ctx.audioWorklet.addModule(url);
- } finally {
- URL.revokeObjectURL(url);
- }
- const node = new AudioWorkletNode(ctx, "mic-pcm", {
- numberOfInputs: 1,
- numberOfOutputs: 1,
- outputChannelCount: [1],
- });
- node.port.onmessage = (e: MessageEvent<{ pcm: Int16Array; last: boolean }>) => {
- gotAudioRef.current = true;
- if (e.data.pcm.length > 0) queueRef.current.push(e.data.pcm);
- if (e.data.last) tailRef.current?.();
- };
- // Web Audio pulls the graph from the destination, so the tap reaches it
- // through a silent gain: the node runs, and nothing echoes the mic.
- const mute = ctx.createGain();
- mute.gain.value = 0;
- const source = ctx.createMediaStreamSource(media);
- source.connect(node);
- node.connect(mute);
- mute.connect(ctx.destination);
- nodesRef.current = { source, node };
- // A context that leaves "running" mid-take stops delivering audio and
- // says nothing; the dictation ends on the state change instead of on a
- // transcript that quietly stopped growing.
- ctx.onstatechange = () => {
- if (ctxRef.current === ctx && ctx.state !== "running") {
- fail("The browser stopped audio capture for this page.");
- }
- };
- } catch (e) {
- const message = e instanceof Error ? e.message : "This browser could not capture the microphone.";
- fail(message);
- return;
- }
-
setStream(media);
-
- feedTimer.current = setInterval(() => void flush(), FEED_MS);
- pollTimer.current = setInterval(async () => {
- const job = jobRef.current;
- if (!job) return;
- try {
- const res = await localBackend.fetch(`/api/cut/mic/${job}`);
- if (!res.ok) return;
- const data = (await res.json()) as { text?: string; status?: string; error?: string };
- if (typeof data.text === "string") setPartial(data.text);
- if (data.status === "error" && data.error) setError(data.error);
- } catch {
- // Transient poll failure; the next tick retries.
- }
- }, POLL_MS);
- silenceTimer.current = setTimeout(() => {
- if (gotAudioRef.current) return;
- fail("No audio is reaching Donkey from this microphone. Check the input device and try again.");
- }, FIRST_AUDIO_MS);
-
+ if (modeRef.current === "engine") {
+ feedTimer.current = setInterval(() => void flush(), FEED_MS);
+ pollTimer.current = setInterval(() => void pollEngine(), POLL_MS);
+ }
+ armSilence();
setState("recording");
- }, [state, flush, fail, teardownAudio]);
+ }, [state, startRecorder, startEngineSession, buildCapture, teardownAudio, flush, pollEngine, armSilence]);
- const confirm = useCallback(async () => {
- const rec = recorderRef.current;
- if (rec) {
- if (state !== "recording") return;
- setState("finishing");
- // Flush the recorder's tail, then transcribe the whole take in one go. A
- // recorder that never reports its stop cannot hold the composer: whatever
- // it has handed over by the deadline is the take.
- const blob = await new Promise<Blob>((resolve) => {
- const settle = () =>
- resolve(new Blob(recChunksRef.current, { type: rec.mimeType || "audio/webm" }));
- const timer = setTimeout(settle, STOP_RECORDER_MS);
- rec.onstop = () => {
- clearTimeout(timer);
- settle();
- };
- try {
- rec.stop();
- } catch {
- clearTimeout(timer);
- settle();
- }
- });
- recorderRef.current = null;
- recChunksRef.current = [];
- teardownAudio();
- if (blob.size === 0) {
- setState("idle");
- setPartial("");
- setError("Nothing was recorded from the microphone.");
- return;
- }
- let text = "";
- try {
- text = await cloudTranscribeRecording(blob, navigator.language);
- } catch (e) {
- setError(e instanceof Error ? e.message : "Transcription failed.");
- }
- setState("idle");
- setPartial("");
- const trimmed = text.trim();
- if (trimmed) onResultRef.current(trimmed);
- return;
- }
+ /** Close the engine's session and take its final transcript. Null when the
+ * engine could not answer, leaving the recording to carry the dictation. */
+ const finishEngine = useCallback(async (): Promise<string | null> => {
const job = jobRef.current;
- if (!job || state !== "recording") return;
- setState("finishing");
+ if (!job) return null;
// Stop pulling new audio, take the chunk the worklet is still filling,
// flush what's queued, then close the input so the model emits its final
// text.
@@ -433,45 +478,94 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro
silenceTimer.current = null;
nodesRef.current?.source.disconnect();
await drainTail();
- // Wait out any in-flight feed, then drain the tail.
+ // Wait out any in-flight feed, then send what the tail added.
while (feedingRef.current) await new Promise((r) => setTimeout(r, 20));
await flush();
- let text = "";
+ jobRef.current = null;
try {
const res = await localBackend.fetch(`/api/cut/mic/${job}/stop`, { method: "POST" });
+ if (!res.ok) return null;
const data = (await res.json().catch(() => ({}))) as { text?: string };
- text = data.text ?? "";
+ return data.text ?? "";
} catch {
- text = partial;
+ void confirmEngine();
+ return null;
}
- jobRef.current = null;
+ }, [drainTail, flush]);
+
+ /** Close the recording and transcribe the whole take on the hosted route. */
+ const finishHosted = useCallback(async (): Promise<string> => {
+ const rec = recorderRef.current;
+ recorderRef.current = null;
+ // Flush the recorder's tail. A recorder that never reports its stop cannot
+ // hold the composer: whatever it has handed over by the deadline is the take.
+ if (rec && rec.state !== "inactive") {
+ await new Promise<void>((resolve) => {
+ const timer = setTimeout(resolve, STOP_RECORDER_MS);
+ rec.onstop = () => {
+ clearTimeout(timer);
+ resolve();
+ };
+ try {
+ rec.stop();
+ } catch {
+ clearTimeout(timer);
+ resolve();
+ }
+ });
+ }
+ const blob = new Blob(recChunksRef.current, { type: rec?.mimeType || "audio/webm" });
+ recChunksRef.current = [];
teardownAudio();
+ if (blob.size === 0) {
+ setError("Nothing was recorded from the microphone.");
+ return "";
+ }
+ try {
+ return await cloudTranscribeRecording(blob, navigator.language);
+ } catch (e) {
+ setError(e instanceof Error ? e.message : "Transcription failed.");
+ // Whatever the engine had said before it dropped out beats nothing.
+ return partialRef.current;
+ }
+ }, [teardownAudio]);
+
+ const confirm = useCallback(async () => {
+ if (state !== "recording") return;
+ setState("finishing");
+ let text: string | null = null;
+ if (modeRef.current === "engine") {
+ text = await finishEngine();
+ // The engine carried it; the recording was only the net under it.
+ if (text !== null) teardownAudio();
+ }
+ if (text === null) text = await finishHosted();
setState("idle");
setPartial("");
+ partialRef.current = "";
const trimmed = text.trim();
if (trimmed) onResultRef.current(trimmed);
- }, [state, flush, partial, teardownAudio, drainTail]);
+ }, [state, finishEngine, finishHosted, teardownAudio]);
const cancel = useCallback(() => {
const job = jobRef.current;
jobRef.current = null;
if (job) void localBackend.fetch(`/api/cut/mic/${job}/cancel`, { method: "POST" }).catch(() => {});
- discardRecorder();
teardownAudio();
setState("idle");
setPartial("");
+ partialRef.current = "";
setError(null);
- }, [discardRecorder, teardownAudio]);
+ }, [teardownAudio]);
// Abandon a dictation if the composer unmounts mid-recording.
useEffect(() => {
return () => {
const job = jobRef.current;
if (job) void localBackend.fetch(`/api/cut/mic/${job}/cancel`, { method: "POST" }).catch(() => {});
- discardRecorder();
teardownAudio();
};
- }, [discardRecorder, teardownAudio]);
+ }, [teardownAudio]);
return { state, stream, partial, error, start, confirm, cancel };
}