Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

7.1 KB
Raw Blame
// Native mono-16k extraction for transcription, in the main process.
//
// The renderer used to do this: `extractMono16kFromVideoUrl` read the whole media
// into a `File`, took an `arrayBuffer()`, handed a `slice(0)` copy to
// `decodeAudioData`, and resampled the result to mono 16k — all of it on the UI
// thread, all of it before whisper ever saw a sample. On a four-minute bed that is
// ~86 MB of decoded float32 plus two copies of the encoded bytes, and it froze the
// editor at open. The inference itself was never the problem: it runs in
// `whisper-stt-server`, in its own process, on the GPU.
//
// So this is the same remedy `useAudioPeaks` already got (see
// `electron/media/audioPeaks.ts`, and the note there about WHICH ffmpeg is packaged
// on Windows): let ffmpeg do it, in the main process, streaming. It costs
// `durationSec * 16000 * 4` bytes — 15.7 MB for that same four-minute bed — and the
// renderer never allocates any of it.
//
// It is deliberately NOT cached on disk, unlike peaks. Peaks are re-read on every
// project open; extraction feeds one transcription, whose RESULT is what gets
// persisted (`document.transcripts[]`). Caching the PCM would trade disk for work
// that is already never repeated.

import { spawn } from "node:child_process";
import { mediaClockAudioArgs, resolveFfmpeg } from "../media/audioPeaks";
import { STT_MEDIA_UNREADABLE } from "./transcriptionContract";

/** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */
const SAMPLE_RATE = 16_000;

/** Past this, it is not a recording — it is a wedged ffmpeg. Matches the peaks path. */
const EXTRACT_TIMEOUT_MS = 60_000;

/**
 * Thrown when no ffmpeg can be resolved. There is no fallback behind it: the renderer
 * decode that used to catch this reads the whole file on the UI thread, and the hidden
 * CLI runner could not read the file at all. Every package ships ffmpeg, so the message
 * names what is missing and how to put it back, for the editor, the CLI and MCP alike.
 */
export class FfmpegUnavailableError extends Error {
	constructor() {
		super(
			"ffmpeg is missing: transcription needs it to read the audio, and this install has none. " +
				"Reinstall OpenScreen, or set OPENSCREEN_FFMPEG_PATH to an ffmpeg executable.",
		);
		this.name = "FfmpegUnavailableError";
	}
}

/** A media with no decodable audio track. Permanent for that file. */
export class NoAudioTrackError extends Error {
	constructor(filePath: string, detail: string) {
		super(`No decodable audio in ${filePath}${detail ? `: ${detail}` : ""}`);
		this.name = "NoAudioTrackError";
	}
}

/**
 * The media file could not be read at all: the OS refused it, or ffmpeg hung
 * opening it. Unlike `NoAudioTrackError` this is NOT permanent: it clears once
 * the user grants access, so it carries `STT_MEDIA_UNREADABLE` for the renderer
 * to explain rather than remember.
 */
export class MediaUnreadableError extends Error {
	constructor(filePath: string, detail: string) {
		super(`${STT_MEDIA_UNREADABLE}: cannot read ${filePath}: ${detail}`);
		this.name = "MediaUnreadableError";
	}
}

/** What ffmpeg's stderr says when the OS refused to open the input. */
const ACCESS_DENIED = /operation not permitted|permission denied/i;

/**
 * Decode `filePath` to mono 16 kHz float samples.
 *
 * Streams `f32le` straight off ffmpeg's stdout, so the only full-size allocation is
 * the result itself. Chunk boundaries do not respect sample boundaries — a 4-byte
 * float can straddle two `data` events — so a partial tail is carried into the next
 * chunk rather than dropped, which would shift every following sample and detune the
 * whole track.
 */
export async function extractMono16kPcm(
	filePath: string,
	options: { signal?: AbortSignal } = {},
): Promise<Float32Array> {
	const ffmpeg = resolveFfmpeg();
	if (!ffmpeg) throw new FfmpegUnavailableError();
	if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError");
	const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath, options.signal);
	// Again: an abort during the probe (which it kills) fired before the listener below.
	if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError");

	const child = spawn(
		ffmpeg,
		[
			"-hide_banner",
			"-loglevel",
			"error",
			"-i",
			filePath,
			...audioArgs,
			"-ac",
			"1",
			"-ar",
			String(SAMPLE_RATE),
			"-f",
			"f32le",
			"-",
		],
		{ stdio: ["ignore", "pipe", "pipe"] },
	);

	return new Promise<Float32Array>((resolve, reject) => {
		const chunks: Float32Array[] = [];
		let total = 0;
		/** Bytes of a float that arrived split across two chunks. */
		let carry: Buffer | null = null;
		let stderr = "";
		let settled = false;

		const finish = (fn: () => void) => {
			if (settled) return;
			settled = true;
			clearTimeout(timer);
			options.signal?.removeEventListener("abort", onAbort);
			fn();
		};

		const timer = setTimeout(() => {
			child.kill("SIGKILL");
			// Only a hang before the first byte says the file could not be opened (a
			// pending macOS folder-access prompt produces exactly that, issue #968). Once
			// audio has come out the media was readable, and the stall is ffmpeg's.
			const nothingRead = total === 0 && carry === null;
			finish(() =>
				reject(
					nothingRead
						? new MediaUnreadableError(filePath, `ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms`)
						: new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`),
				),
			);
		}, EXTRACT_TIMEOUT_MS);

		const onAbort = () => {
			child.kill("SIGKILL");
			finish(() => reject(new DOMException("Aborted", "AbortError")));
		};
		options.signal?.addEventListener("abort", onAbort, { once: true });

		child.stdout.on("data", (c: Buffer) => {
			const buf = carry ? Buffer.concat([carry, c]) : c;
			const usable = buf.length - (buf.length % 4);
			if (usable > 0) {
				// Copy rather than view: a Buffer's memory is rarely 4-byte aligned, and
				// `byteOffset` is almost never 0.
				const view = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + usable));
				chunks.push(view);
				total += view.length;
			}
			carry = usable < buf.length ? Buffer.from(buf.subarray(usable)) : null;
		});
		child.stderr.on("data", (c: Buffer) => {
			stderr = (stderr + c.toString()).slice(-2048);
		});
		child.once("error", (err) => finish(() => reject(err)));
		child.once("close", (code) => {
			// A file with no audio track exits non-zero, and so does a corrupt one. The
			// caller treats both the same way — this asset will not transcribe — so they
			// share an error type; `stderr` carries which it was.
			if (code !== 0 && total === 0) {
				// Except a read the OS refused: that also exits non-zero with nothing
				// decoded, but remembering it as "no audio" would never retry the file
				// once access is granted.
				const error = ACCESS_DENIED.test(stderr)
					? new MediaUnreadableError(filePath, stderr.trim())
					: new NoAudioTrackError(filePath, stderr.trim());
				finish(() => reject(error));
				return;
			}
			const out = new Float32Array(total);
			let at = 0;
			for (const part of chunks) {
				out.set(part, at);
				at += part.length;
			}
			finish(() => resolve(out));
		});
	});
}