Open source CapCut alternative.
TypeScript
fix(site): dictation captures on a worklet and says when it stops hearing
David Le202206028588
diff --git a/site/src/cut/components/MicDictation.tsx b/site/src/cut/components/MicDictation.tsxindex d5a0cc84..304e94db 100644--- a/site/src/cut/components/MicDictation.tsx+++ b/site/src/cut/components/MicDictation.tsx@@ -35,6 +35,9 @@ export function DictationBody({}) {const shown = [text.trim(), mic.partial].filter(Boolean).join(" ");const finishing = mic.state === "finishing";+ // Confirm only means something once capture is live; while the graph is+ // still coming up the button says so instead of swallowing the click.+ const ready = mic.state === "recording";// Keyboard control while dictating: Enter confirms, Escape cancels. Capture// on window so an input still focused under the overlay never sees the key.useEffect(() => {@@ -69,7 +72,11 @@ export function DictationBody({className="h-7 min-w-0 flex-1 text-muted-foreground/70"/>) : (- <div className="flex-1" />+ // Without the app the take is transcribed after the stop, which is a+ // round trip the meter's empty slot can say out loud.+ <div className="flex-1 px-1 text-[11px] text-muted-foreground/70">+ {finishing ? "Transcribing…" : null}+ </div>)}<buttontype="button"@@ -83,7 +90,7 @@ export function DictationBody({<buttontype="button"title="Use transcription"- disabled={finishing}+ disabled={!ready}onClick={() => void mic.confirm()}className="grid size-7 place-items-center rounded-lg bg-[#0a84ff] text-white transition-colors hover:bg-[#0a84ff]/90 disabled:opacity-50">diff --git a/site/src/cut/lib/micTranscribe.ts b/site/src/cut/lib/micTranscribe.tsindex 8ee08ee5..35135df2 100644--- a/site/src/cut/lib/micTranscribe.ts+++ b/site/src/cut/lib/micTranscribe.ts@@ -12,28 +12,78 @@ import { cloudTranscribeRecording } from "./cloudTranscribe";// On the cloud backend there is no engine: the mic records locally// (MediaRecorder) and one hosted transcription runs when the user confirms —// no live partials, so the composer shows "Listening…" while recording.+//+// Capture is an AudioWorklet. It downsamples on the audio thread and hands the+// main thread whole chunks, and it releases its partial chunk on request so the+// end of the last word reaches the model. A dictation that stops carrying audio+// — a graph that never started, feeds the engine refuses — ends with the reason+// on screen, because a session the engine never hears from produces nothing and+// waits for a transcript that will never come.export type MicState = "idle" | "starting" | "recording" | "finishing";const TARGET_RATE = 16000; // must match cut-stt.swift's live input format+const CHUNK_MS = 250; // how much PCM the worklet batches into one messageconst FEED_MS = 250; // how often queued PCM is flushed to the engineconst POLL_MS = 150; // how often the evolving transcript is fetched+// Capture that has delivered nothing by here never will: the graph is dead, and+// waiting longer turns a failure the user can act on into a silent one. Both+// paths are held to it — the worklet's first chunk, the recorder's first blob.+const FIRST_AUDIO_MS = 2500;+// How often MediaRecorder hands over what it has on the hosted path.+const RECORDER_SLICE_MS = 1000;+// How long that recorder may take to report its stop before the take is read+// from the blobs already in hand.+const STOP_RECORDER_MS = 1500;+// Feeds drop transiently; a run of them means the audio is going nowhere.+const MAX_FEED_FAILURES = 3;+// How long the worklet's tail chunk may take once the flush is asked for.+const TAIL_MS = 400;-/** Average-downsample Float32 PCM to 16 kHz signed-16-bit little-endian. */-function toPcm16(input: Float32Array, srcRate: number): Int16Array {- const ratio = srcRate / TARGET_RATE;- const outLen = Math.floor(input.length / ratio);- const out = new Int16Array(outLen);- for (let i = 0; i < outLen; i++) {- const start = Math.floor(i * ratio);- const end = Math.min(input.length, Math.floor((i + 1) * ratio));- let sum = 0;- for (let j = start; j < end; j++) sum += input[j];- const v = Math.max(-1, Math.min(1, sum / Math.max(1, end - start)));- out[i] = v < 0 ? v * 0x8000 : v * 0x7fff;+/** The capture graph, as an AudioWorklet module. It averages the mic's native+ * rate down to 16 kHz signed-16-bit PCM on the audio thread and posts whole+ * chunks; a "flush" message releases the partial chunk that is still forming,+ * so the tail of the take is not dropped at the cut. */+const PCM_WORKLET = `+class MicPcm extends AudioWorkletProcessor {+ constructor() {+ super();+ this.ratio = sampleRate / ${TARGET_RATE};+ this.size = Math.round((${TARGET_RATE} * ${CHUNK_MS}) / 1000);+ this.out = new Int16Array(this.size);+ this.filled = 0;+ this.sum = 0;+ this.count = 0;+ this.acc = 0;+ this.port.onmessage = (e) => {+ if (e.data === "flush") this.send(true);+ };+ }+ send(last) {+ const pcm = this.out.slice(0, this.filled);+ this.filled = 0;+ this.port.postMessage({ pcm, last }, [pcm.buffer]);+ }+ process(inputs) {+ const ch = inputs[0] && inputs[0][0];+ if (!ch) return true;+ for (let i = 0; i < ch.length; i++) {+ this.sum += ch[i];+ this.count++;+ this.acc++;+ if (this.acc < this.ratio) continue;+ this.acc -= this.ratio;+ const v = Math.max(-1, Math.min(1, this.sum / this.count));+ this.out[this.filled++] = v < 0 ? v * 0x8000 : v * 0x7fff;+ this.sum = 0;+ this.count = 0;+ if (this.filled === this.size) this.send(false);+ }+ return true;}- return out;}+registerProcessor("mic-pcm", MicPcm);+`;export interface MicController {state: MicState;@@ -61,11 +111,18 @@ export function useMicTranscription(onResult: (text: string) => void): MicControconst recChunksRef = useRef<Blob[]>([]);const streamRef = useRef<MediaStream | null>(null);const ctxRef = useRef<AudioContext | null>(null);- const nodesRef = useRef<{ source: MediaStreamAudioSourceNode; processor: ScriptProcessorNode } | null>(null);+ const nodesRef = useRef<{ source: MediaStreamAudioSourceNode; node: AudioWorkletNode } | null>(null);const queueRef = useRef<Int16Array[]>([]);const feedingRef = useRef(false);+ const failuresRef = useRef(0);+ /** Whether capture has delivered anything at all — the liveness test that+ * separates a quiet room from a capture that never ran. */+ const gotAudioRef = useRef(false);+ /** Resolves the pending tail flush when the worklet's last chunk arrives. */+ const tailRef = useRef<(() => void) | null>(null);const feedTimer = useRef<ReturnType<typeof setInterval> | null>(null);const pollTimer = useRef<ReturnType<typeof setInterval> | null>(null);+ const silenceTimer = useRef<ReturnType<typeof setTimeout> | null>(null);const onResultRef = useRef(onResult);useLayoutEffect(() => {onResultRef.current = onResult;@@ -88,20 +145,51 @@ export function useMicTranscription(onResult: (text: string) => void): MicControconst teardownAudio = useCallback(() => {if (feedTimer.current) clearInterval(feedTimer.current);if (pollTimer.current) clearInterval(pollTimer.current);+ if (silenceTimer.current) clearTimeout(silenceTimer.current);feedTimer.current = null;pollTimer.current = null;- nodesRef.current?.processor.disconnect();- nodesRef.current?.source.disconnect();+ silenceTimer.current = null;+ tailRef.current = null;+ const nodes = nodesRef.current;nodesRef.current = null;- void ctxRef.current?.close().catch(() => {});+ if (nodes) {+ nodes.node.port.onmessage = null;+ nodes.node.disconnect();+ nodes.source.disconnect();+ }+ const ctx = ctxRef.current;ctxRef.current = null;+ if (ctx) {+ ctx.onstatechange = null;+ void ctx.close().catch(() => {});+ }streamRef.current?.getTracks().forEach((t) => t.stop());streamRef.current = null;queueRef.current = [];feedingRef.current = false;+ failuresRef.current = 0;+ gotAudioRef.current = false;setStream(null);}, []);+ /** Hand the session back and put the reason on screen. Everything that can+ * leave a dictation unable to carry audio ends here, so the engine never+ * holds a session nothing is feeding and the user is never left watching an+ * empty transcript. */+ const fail = useCallback(+ (message: string) => {+ const job = jobRef.current;+ jobRef.current = null;+ if (job) void localBackend.fetch(`/api/cut/mic/${job}/cancel`, { method: "POST" }).catch(() => {});+ discardRecorder();+ teardownAudio();+ setState("idle");+ setPartial("");+ setError(message);+ },+ [discardRecorder, teardownAudio]+ );+/** Concatenate and POST all queued PCM. Guarded so chunks stay ordered. */const flush = useCallback(async () => {const job = jobRef.current;@@ -117,16 +205,41 @@ export function useMicTranscription(onResult: (text: string) => void): MicControoff += c.length;}try {- await localBackend.fetch(`/api/cut/mic/${job}/feed`, {+ const res = await localBackend.fetch(`/api/cut/mic/${job}/feed`, {method: "POST",headers: { "Content-Type": "application/octet-stream" },body: merged.buffer,});- } catch {- // A dropped chunk just shortens the transcript; keep the dictation alive.+ if (!res.ok) throw new Error(`Donkey turned the audio away (${res.status}).`);+ failuresRef.current = 0;+ } catch (e) {+ // A dropped chunk just shortens the transcript; a run of them means the+ // audio is reaching nothing, and the dictation says so.+ failuresRef.current += 1;+ if (failuresRef.current >= MAX_FEED_FAILURES) {+ fail(e instanceof Error ? e.message : "Donkey stopped receiving the audio.");+ }} finally {feedingRef.current = false;}+ }, [fail]);++ /** Ask the worklet for the chunk still forming, so the last word is whole. */+ const drainTail = useCallback(async () => {+ const node = nodesRef.current?.node;+ if (!node) return;+ await new Promise<void>((resolve) => {+ const timer = setTimeout(() => {+ tailRef.current = null;+ resolve();+ }, TAIL_MS);+ tailRef.current = () => {+ clearTimeout(timer);+ tailRef.current = null;+ resolve();+ };+ node.port.postMessage("flush");+ });}, []);const start = useCallback(async () => {@@ -142,56 +255,103 @@ export function useMicTranscription(onResult: (text: string) => void): MicControsetError("Microphone access was blocked. Allow the microphone for this site, then try again.");return;}+ streamRef.current = media;// Dictation runs on the Mac whenever the app is there — on-device, free,// and live — whatever backend holds the project. Without it the take is- // recorded and sent to the hosted route in one go.+ // recorded and sent to the hosted route in one go, under the same contract:+ // a recorder that cannot start, that dies mid-take, or that hands over+ // nothing ends the dictation with the reason on screen.if (!hasLocalCompute()) {- recChunksRef.current = [];- const rec = new MediaRecorder(media);- rec.ondataavailable = (e) => {- if (e.data.size > 0) recChunksRef.current.push(e.data);- };- rec.start(1000);- recorderRef.current = rec;- streamRef.current = media;+ try {+ recChunksRef.current = [];+ const rec = new MediaRecorder(media);+ rec.ondataavailable = (e) => {+ if (e.data.size === 0) return;+ gotAudioRef.current = true;+ recChunksRef.current.push(e.data);+ };+ rec.onerror = () => fail("The browser stopped recording the microphone.");+ rec.start(RECORDER_SLICE_MS);+ recorderRef.current = rec;+ } catch (e) {+ teardownAudio();+ setState("idle");+ setError(e instanceof Error ? e.message : "This browser could not record the microphone.");+ return;+ }setStream(media);+ silenceTimer.current = setTimeout(() => {+ if (gotAudioRef.current) return;+ fail("No audio is reaching Donkey from this microphone. Check the input device and try again.");+ }, FIRST_AUDIO_MS);setState("recording");return;}+ let jobId: string;try {const res = await localBackend.fetch("/api/cut/mic/start", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({}),+ body: JSON.stringify({ locale: navigator.language }),});const data = (await res.json().catch(() => ({}))) as { id?: string; error?: string };if (!res.ok || !data.id) throw new Error(data.error || "Dictation needs the Donkey app running on this Mac.");- jobRef.current = data.id;+ jobId = data.id;+ jobRef.current = jobId;} catch (e) {- media.getTracks().forEach((t) => t.stop());+ teardownAudio();setState("idle");setError(e instanceof Error ? e.message : "Could not start dictation.");return;}- streamRef.current = media;- setStream(media);+ // The engine now holds a session, so a capture that cannot be built hands+ // it back here instead of leaving a process waiting on audio.+ try {+ const ctx = new AudioContext();+ ctxRef.current = ctx;+ await ctx.resume();+ if (ctx.state !== "running") throw new Error("This page is not allowed to capture audio right now.");+ const url = URL.createObjectURL(new Blob([PCM_WORKLET], { type: "text/javascript" }));+ try {+ await ctx.audioWorklet.addModule(url);+ } finally {+ URL.revokeObjectURL(url);+ }+ const node = new AudioWorkletNode(ctx, "mic-pcm", {+ numberOfInputs: 1,+ numberOfOutputs: 1,+ outputChannelCount: [1],+ });+ node.port.onmessage = (e: MessageEvent<{ pcm: Int16Array; last: boolean }>) => {+ gotAudioRef.current = true;+ if (e.data.pcm.length > 0) queueRef.current.push(e.data.pcm);+ if (e.data.last) tailRef.current?.();+ };+ // Web Audio pulls the graph from the destination, so the tap reaches it+ // through a silent gain: the node runs, and nothing echoes the mic.+ const mute = ctx.createGain();+ mute.gain.value = 0;+ const source = ctx.createMediaStreamSource(media);+ source.connect(node);+ node.connect(mute);+ mute.connect(ctx.destination);+ nodesRef.current = { source, node };+ // A context that leaves "running" mid-take stops delivering audio and+ // says nothing; the dictation ends on the state change instead of on a+ // transcript that quietly stopped growing.+ ctx.onstatechange = () => {+ if (ctxRef.current === ctx && ctx.state !== "running") {+ fail("The browser stopped audio capture for this page.");+ }+ };+ } catch (e) {+ const message = e instanceof Error ? e.message : "This browser could not capture the microphone.";+ fail(message);+ return;+ }- const ctx = new AudioContext();- void ctx.resume().catch(() => {});- ctxRef.current = ctx;- const source = ctx.createMediaStreamSource(media);- const processor = ctx.createScriptProcessor(4096, 1, 1);- processor.onaudioprocess = (e) => {- queueRef.current.push(toPcm16(e.inputBuffer.getChannelData(0), ctx.sampleRate));- };- // Route through a muted gain so onaudioprocess fires without echoing the mic.- const mute = ctx.createGain();- mute.gain.value = 0;- source.connect(processor);- processor.connect(mute);- mute.connect(ctx.destination);- nodesRef.current = { source, processor };+ setStream(media);feedTimer.current = setInterval(() => void flush(), FEED_MS);pollTimer.current = setInterval(async () => {@@ -207,24 +367,46 @@ export function useMicTranscription(onResult: (text: string) => void): MicContro// Transient poll failure; the next tick retries.}}, POLL_MS);+ silenceTimer.current = setTimeout(() => {+ if (gotAudioRef.current) return;+ fail("No audio is reaching Donkey from this microphone. Check the input device and try again.");+ }, FIRST_AUDIO_MS);setState("recording");- }, [state, flush]);+ }, [state, flush, fail, teardownAudio]);const confirm = useCallback(async () => {const rec = recorderRef.current;if (rec) {if (state !== "recording") return;setState("finishing");- // Flush the recorder's tail, then transcribe the whole take in one go.+ // Flush the recorder's tail, then transcribe the whole take in one go. A+ // recorder that never reports its stop cannot hold the composer: whatever+ // it has handed over by the deadline is the take.const blob = await new Promise<Blob>((resolve) => {- rec.onstop = () =>+ const settle = () =>resolve(new Blob(recChunksRef.current, { type: rec.mimeType || "audio/webm" }));- rec.stop();+ const timer = setTimeout(settle, STOP_RECORDER_MS);+ rec.onstop = () => {+ clearTimeout(timer);+ settle();+ };+ try {+ rec.stop();+ } catch {+ clearTimeout(timer);+ settle();+ }});recorderRef.current = null;recChunksRef.current = [];teardownAudio();+ if (blob.size === 0) {+ setState("idle");+ setPartial("");+ setError("Nothing was recorded from the microphone.");+ return;+ }let text = "";try {text = await cloudTranscribeRecording(blob, navigator.language);@@ -240,13 +422,17 @@ export function useMicTranscription(onResult: (text: string) => void): MicControconst job = jobRef.current;if (!job || state !== "recording") return;setState("finishing");- // Stop pulling new audio, flush what's queued, then close the input so the- // model emits its final text.- nodesRef.current?.processor.disconnect();+ // Stop pulling new audio, take the chunk the worklet is still filling,+ // flush what's queued, then close the input so the model emits its final+ // text.if (feedTimer.current) clearInterval(feedTimer.current);if (pollTimer.current) clearInterval(pollTimer.current);+ if (silenceTimer.current) clearTimeout(silenceTimer.current);feedTimer.current = null;pollTimer.current = null;+ silenceTimer.current = null;+ nodesRef.current?.source.disconnect();+ await drainTail();// Wait out any in-flight feed, then drain the tail.while (feedingRef.current) await new Promise((r) => setTimeout(r, 20));await flush();@@ -264,7 +450,7 @@ export function useMicTranscription(onResult: (text: string) => void): MicControsetPartial("");const trimmed = text.trim();if (trimmed) onResultRef.current(trimmed);- }, [state, flush, partial, teardownAudio]);+ }, [state, flush, partial, teardownAudio, drainTail]);const cancel = useCallback(() => {const job = jobRef.current;