Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

23.0 KB
Raw Blame
import path from "node:path";
import { app, type IpcMain, net } from "electron";
import { planChunks } from "./chunking";
import { extractMono16kPcm } from "./extractAudio";
import {
	alignerPath,
	cachedAligner,
	ensureAligner,
	ensureModels,
	modelPaths,
} from "./modelManager";
import type {
	SttPhraseSegment,
	SttStatusEvent,
	SttTiming,
	SttTranscribeRequest,
	SttTranscribeResponse,
	SttWordSegment,
} from "./transcriptionContract";
import { WhisperServerManager } from "./whisperServer";

/**
 * Owner of the long-lived STT pipeline. One instance per Electron app.
 *
 * Workflow:
 *   1. `init()` spawns `whisper-stt-server` (or queues the call if it's busy).
 *   2. `transcribe()` splits the renderer's `Float32Array` into chunks
 *      (`chunking.ts`) and runs each through whisper-stt-server's HTTP
 *      `/inference`, which returns both phrase- and word-level segments in one
 *      pass (see whisperServer.ts). Word timestamps come from whisper.cpp's
 *      native DTW token timestamps (`t_dtw`, SMALL aheads preset,
 *      `flash_attn = false`), see
 *      technical-documentation/architecture/transcription-and-captions.md § Decision rationale.
 *   3. `shutdown()` tears down on app quit.
 *
 * Status events fan out to every attached sink so the renderer can drive its
 * "loading model" / "transcribing" indicator.
 *
 * Why chunked rather than one request: a 30-minute recording took ~10 minutes
 * in a single `/inference` call — no progress to show, no way to recover from a
 * transient failure without redoing everything, and long enough that the HTTP
 * client's own header timeout killed it before whisper ever answered. Chunks
 * turn that into a progress signal, a retry unit, and — via `cancel()` — the
 * only point where a run in flight can be stopped at all.
 *
 * Chunks run SEQUENTIALLY, and that is a measured choice, not an omission:
 * whisper-stt-server holds a single model context, so concurrent `/inference`
 * calls don't just serialize — they get SLOWER. Two 120s chunks took 76.9s one
 * after the other and 144.1s fired together (0.53x, i.e. ~1.9x slower) on this
 * Vulkan backend. A client-side worker pool is therefore a pessimisation. Real
 * parallelism would need several server processes, each with its own copy of
 * the model resident on the GPU; that trade (VRAM + spawn cost per worker) is
 * worth revisiting only if a much smaller model ever becomes the default.
 */

/** What `extractMono16kPcm` decodes to (see `extractAudio.ts`). */
const SAMPLE_RATE = 16_000;

/** Attempts per chunk before the whole transcription fails. */
const CHUNK_ATTEMPTS = 3;

/**
 * Model and aligner downloads go through Chromium's network stack rather than Node's
 * global `fetch`, which ignores the system proxy: where huggingface.co is only reachable
 * through one (mainland China, many corporate networks), the download failed with a bare
 * "fetch failed" while the same URL opened in a browser (#1330).
 */
const fetchThroughChromium: typeof fetch = (input, init) =>
	net.fetch(input instanceof URL ? input.href : input, init);

/**
 * `AbortError` by name so the renderer's `isAbortError` recognizes it as "the
 * user asked for this" rather than an engine failure worth a toast.
 */
function cancelledError(): Error {
	const error = new Error("Transcription cancelled");
	error.name = "AbortError";
	return error;
}

/**
 * One transcription's cost, in both of the conventions this repo uses.
 *
 * whisper.cpp and the POC report measure RTF — wall-clock divided by audio, so
 * LOWER is faster — while the figure that means something to a reader is its
 * reciprocal ("2.1x real-time"). Printing both lets a log line be compared
 * against tools/stt-eval/whispercpp-dtw-poc/REPORT.md 5 without arithmetic.
 *
 * ASCII "x" rather than the report's "×" glyph: these lines are read in a
 * Windows console and in the diagnostics dump, neither of which is reliably
 * UTF-8.
 */
function formatTiming(timing: SttTiming): string {
	const speed = timing.rtf > 0 ? 1 / timing.rtf : 0;
	return (
		`${timing.audioSec.toFixed(1)}s audio in ${timing.elapsedSec.toFixed(1)}s ` +
		`(${timing.rtf.toFixed(2)} rtf, ${speed.toFixed(1)}x real-time)`
	);
}

export interface SttManagerInitOptions {
	statusSink?: (event: SttStatusEvent) => void;
	/** Override the models cache directory; defaults to `app.getPath("userData") + "/stt-models"`. */
	modelsBaseDir?: string;
}

export class SttManager {
	private readonly server = new WhisperServerManager();
	private shuttingDown = false;
	private modelsBaseDir: string | null = null;
	private readonly statusSinks = new Set<(event: SttStatusEvent) => void>();
	private initPromise: Promise<void> | null = null;
	/** Kept from `prepare()` so a chunk retry can respawn a helper that died mid-run. */
	private modelPath: string | null = null;
	private vadModelPath: string | null = null;
	/**
	 * Bumped by `cancel()`. The chunk loop compares it against the value it
	 * captured on entry, so a cancel that lands after a new run started cannot
	 * kill that new run.
	 */
	private cancelEpoch = 0;

	/**
	 * The word aligner per language (ctcAlign.ts). A chunk never waits for a
	 * download: it is an improvement, and a 350 MB fetch inside a chunk would hold
	 * the helper's queue for as long as the network takes, past any request
	 * ceiling and out of Cancel's reach. So:
	 * - a copy already on disk is verified in place the first time a language is
	 *   asked for in a session (a local read, no network), then reused;
	 * - otherwise the download starts in the background and the chunks keep
	 *   whisper's times until it lands; later chunks and transcriptions use it;
	 * - Cancel and quit abort it, and so does a stall (`ensureAligner`). A failed
	 *   one is not retried for the rest of the run, only on the next transcription.
	 */
	private readonly alignersReady = new Map<string, string>();
	private readonly alignersChecked = new Set<string>();
	private readonly alignerDownloads = new Map<string, AbortController>();
	private readonly alignersFailed = new Set<string>();

	private readonly alignerFor = async (language: string): Promise<string | null> => {
		const ready = this.alignersReady.get(language);
		if (ready) return ready;
		if (!alignerPath(this.getModelsDir(), language)) return null;
		if (!this.alignersChecked.has(language)) {
			this.alignersChecked.add(language);
			const cached = await cachedAligner(this.getModelsDir(), language);
			if (cached) {
				this.alignersReady.set(language, cached);
				return cached;
			}
		}
		if (!this.alignersFailed.has(language) && !this.alignerDownloads.has(language)) {
			this.downloadAligner(language);
		}
		return null;
	};

	private downloadAligner(language: string): void {
		const controller = new AbortController();
		this.alignerDownloads.set(language, controller);
		console.info(`[stt] downloading the "${language}" word aligner in the background`);
		ensureAligner({
			baseDir: this.getModelsDir(),
			language,
			signal: controller.signal,
			fetcher: fetchThroughChromium,
		})
			.then((file) => {
				if (file) this.alignersReady.set(language, file);
				console.info(`[stt] "${language}" word aligner ready`);
			})
			.catch((error: unknown) => {
				this.alignersFailed.add(language);
				console.warn(
					`[stt] no word aligner for "${language}", keeping whisper's word times: ` +
						`${error instanceof Error ? error.message : String(error)}`,
				);
			})
			.finally(() => this.alignerDownloads.delete(language));
	}

	/**
	 * The extraction in flight, if any. `cancelEpoch` alone stops the CHUNK loop, which is
	 * checked between chunks — so a cancel during the decode left ffmpeg running to
	 * completion on a file that can be hours long, and the user saw nothing stop.
	 */
	private extraction: AbortController | null = null;

	/**
	 * Attach a sink for the renderer status channel; returns its detach function.
	 *
	 * A SET rather than one slot: this used to be a single field that each IPC
	 * invocation saved and restored, so with two overlapping transcriptions the
	 * first to finish restored the sink captured at ITS start and left the other
	 * one emitting into nothing — no progress for the rest of its run, which
	 * reads as a hang.
	 */
	addStatusSink(sink: (event: SttStatusEvent) => void): () => void {
		this.statusSinks.add(sink);
		return () => {
			this.statusSinks.delete(sink);
		};
	}

	private emit(event: SttStatusEvent): void {
		for (const sink of this.statusSinks) sink(event);
	}

	/**
	 * Stop the in-flight transcription at the next chunk boundary.
	 *
	 * ponytail: one epoch for the whole manager, not a handle per request. The
	 * pipeline runs one transcription at a time by construction (the renderer's
	 * queue serializes, and `WhisperServerManager` single-flights on top), so
	 * "cancel what is running" is the only question anyone can ask. Per-request
	 * tokens the day two recordings can transcribe at once.
	 */
	cancel(): void {
		this.cancelEpoch++;
		this.extraction?.abort();
		for (const download of this.alignerDownloads.values()) download.abort(cancelledError());
	}

	/**
	 * Run all one-time setup; cheap to call repeatedly — the `initPromise`
	 * means the second caller just awaits the same completion.
	 */
	init(options: SttManagerInitOptions = {}): Promise<void> {
		if (this.shuttingDown) return Promise.reject(cancelledError());
		if (options.statusSink) this.addStatusSink(options.statusSink);
		if (options.modelsBaseDir) this.modelsBaseDir = options.modelsBaseDir;
		if (!this.initPromise) {
			// A REJECTED init must not be cached. `prepare()` downloads a 264 MB
			// model on first run, and caching its rejection meant one dropped
			// connection poisoned the whole app session: every later transcription
			// — including the retry the UI offers, and every remaining asset in the
			// auto-transcription queue — awaited the same stale rejection and failed
			// in milliseconds, with no way back short of quitting the app.
			// Reconnecting the network changed nothing. Clearing the slot on failure
			// makes the next attempt a real attempt.
			this.initPromise = this.prepare().catch((error) => {
				this.initPromise = null;
				throw error;
			});
		}
		return this.initPromise;
	}

	private getModelsDir(): string {
		if (this.modelsBaseDir) return this.modelsBaseDir;
		this.modelsBaseDir = path.join(app.getPath("userData"), "stt-models");
		return this.modelsBaseDir;
	}

	private async prepare(): Promise<void> {
		const modelsDir = this.getModelsDir();
		this.emit({ phase: "model", model: "whisper", downloadedBytes: 0, totalBytes: 0 });
		await ensureModels({
			baseDir: modelsDir,
			fetcher: fetchThroughChromium,
			onProgress: (event) => {
				this.emit({
					phase: "model",
					model: event.id,
					downloadedBytes: event.downloadedBytes,
					totalBytes: event.totalBytes,
				});
			},
		});
		if (this.shuttingDown) throw cancelledError();

		const paths = modelPaths(modelsDir);
		this.modelPath = paths.whisper;
		this.vadModelPath = paths["silero-vad"];
		try {
			await this.server.start({
				modelPath: paths.whisper,
				vadModelPath: paths["silero-vad"],
			});
		} catch (error) {
			if (this.shuttingDown) throw cancelledError();
			throw error;
		}
		if (this.shuttingDown) {
			await this.server.shutdown();
			throw cancelledError();
		}
		this.emit({ phase: "transcribe" });
	}

	/**
	 * Run one chunk, retrying a few times before giving up on the whole request.
	 *
	 * A failure here is usually the helper process dying (OOM, driver reset)
	 * rather than a bad chunk, so each retry first re-runs `server.start()` —
	 * idempotent when the helper is alive, a respawn when it isn't. That is what
	 * makes a 30-minute transcription survive a helper that dies once mid-run.
	 *
	 * What it does NOT do is salvage a chunk that fails all three attempts: the
	 * request fails whole and the chunks that already succeeded go with it. A
	 * transcript silently missing 90 seconds in the middle is worse than no
	 * transcript, since nothing downstream (captions, trims, the transcript
	 * editor) could tell the gap from a silence. The caller is told how far it
	 * got instead — see the wrapper in `transcribe()`.
	 */
	private async transcribeChunk(
		samples: Float32Array,
		language: string | undefined,
		prompt: string | undefined,
	): Promise<Awaited<ReturnType<WhisperServerManager["transcribe"]>>> {
		let lastError: unknown;
		for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) {
			if (this.shuttingDown) throw cancelledError();
			try {
				return await this.server.transcribe({
					samples,
					language,
					prompt,
					alignerFor: this.alignerFor,
				});
			} catch (error) {
				lastError = error;
				if (this.shuttingDown) throw cancelledError();
				if (attempt === CHUNK_ATTEMPTS) break;
				if (this.modelPath) {
					await this.server
						.start({ modelPath: this.modelPath, vadModelPath: this.vadModelPath })
						.catch(() => undefined);
				}
				if (this.shuttingDown) throw cancelledError();
				await new Promise((resolve) => setTimeout(resolve, 500 * attempt));
			}
		}
		throw lastError instanceof Error ? lastError : new Error(String(lastError));
	}

	/** Transcribe a whole recording, chunk by chunk, reporting progress as it goes. */
	/** Decode `sourcePath` into the samples `transcribe` needs. */
	private async extract(req: SttTranscribeRequest): Promise<Float32Array> {
		if (!req.sourcePath) {
			throw new Error("stt:transcribe needs either `samples` or `sourcePath`");
		}
		const controller = new AbortController();
		this.extraction = controller;
		try {
			return await extractMono16kPcm(req.sourcePath, { signal: controller.signal });
		} finally {
			// Only if it is still ours: a cancel that started a new run must not have its
			// controller cleared by the old one unwinding.
			if (this.extraction === controller) this.extraction = null;
		}
	}

	async transcribe(req: SttTranscribeRequest): Promise<SttTranscribeResponse> {
		await this.init();

		const epoch = this.cancelEpoch;
		this.alignersFailed.clear();
		// Extraction is part of the run, and on a long file it is the part the user used
		// to watch the editor freeze through. Doing it here means the renderer hands over
		// a path and gets segments back, holding none of the audio. No new status phase:
		// the caller already reports "extracting-audio" around this call, and the work
		// simply moved to the other side of the IPC.
		const samples = req.samples ?? (await this.extract(req));
		const totalSec = samples.length / SAMPLE_RATE;
		const chunks = planChunks(samples, SAMPLE_RATE);
		this.emit({ phase: "transcribe", completedSec: 0, totalSec });

		const segments: SttPhraseSegment[] = [];
		const wordSegments: SttWordSegment[] = [];
		let detectedLanguage: string | null = null;
		let backend = this.server.status.backend ?? "whispercpp-cpu";
		// Summed, not averaged: chunks differ in length, so the mean of the
		// per-chunk RTFs is not the RTF of the run.
		let elapsedSec = 0;
		let audioSec = 0;
		let timedChunks = 0;
		let untimedChunks = 0;
		// The CPU verdict is worth saying once per run, not once per chunk.
		let warnedCpu = false;
		// Only the first chunk auto-detects; every later chunk is forced onto the
		// language it resolved, so whisper cannot flip mid-recording on a chunk
		// that opens with a proper noun or a silence and "transcribe" the rest as
		// another language.
		//
		// `"auto"` must collapse to `undefined` here rather than merely falsy
		// values: `SttTranscribeRequest` documents it as the explicit way to ask
		// for detection, and it is TRUTHY — left in place it makes the pin below
		// unreachable for every caller that spells its intent out.
		//
		// This depends on the helper reporting what it RESOLVED rather than
		// echoing the request, which it only does since cc781806 (30/07/2026,
		// `whisper_full_lang_id()` in electron/native/whisper-stt/src/main.cpp).
		// A stale `electron/native/bin/<tag>/whisper-stt-server` — the directory
		// is gitignored, so a dev tree keeps whatever was last staged there —
		// silently reverts this to "every chunk detects on its own": the echo
		// comes back as the literal "auto", the guard below rejects it, and
		// nothing anywhere says why. `scripts/stage-whisper-stt.sh` refuses to
		// overwrite a local binary by design, so it will not rescue you either.
		// Verified end-to-end on 352s of real speech: `[undefined,"en","en","en"]`.
		let language = req.language && req.language !== "auto" ? req.language : undefined;

		for (const [index, chunk] of chunks.entries()) {
			// Between chunks is the only place this loop can be interrupted, and it
			// is enough: a chunk is bounded by `whisperServer`'s own request ceiling.
			if (this.cancelEpoch !== epoch) throw cancelledError();
			const offsetSec = chunk.startSample / SAMPLE_RATE;
			const result = await this.transcribeChunk(
				samples.subarray(chunk.startSample, chunk.endSample),
				language,
				req.prompt,
			).catch((error) => {
				if (error instanceof Error && error.name === "AbortError") throw error;
				// Say where it died. Without this the user gets "Transcription
				// failed" for a 30-minute recording with no hint that 18 of those
				// minutes were fine and the helper fell over at one specific spot.
				// `Object.assign` rather than the `{ cause }` constructor option: the
				// project targets ES2020, where that overload does not exist (see
				// `BackgroundLoadError` in src/lib/wallpaper.ts for the same dance).
				throw Object.assign(
					new Error(
						`transcription failed ${Math.round(offsetSec)}s into a ${Math.round(totalSec)}s ` +
							`recording (chunk ${index + 1}/${chunks.length}): ` +
							`${error instanceof Error ? error.message : String(error)}`,
					),
					{ cause: error },
				);
			});
			if (this.shuttingDown || this.cancelEpoch !== epoch) throw cancelledError();
			// Chunk-relative timestamps → absolute, the only thing every consumer
			// (captions, transcript editor, trims) reads.
			for (const segment of result.segments) {
				segments.push({
					text: segment.text,
					startSec: segment.startSec + offsetSec,
					endSec: segment.endSec + offsetSec,
				});
			}
			for (const word of result.wordSegments) {
				wordSegments.push({
					word: word.word,
					startSec: word.startSec + offsetSec,
					endSec: word.endSec + offsetSec,
					confidence: word.confidence,
				});
			}
			if (!detectedLanguage && result.detectedLanguage && result.detectedLanguage !== "auto") {
				detectedLanguage = result.detectedLanguage;
				if (!language) language = detectedLanguage;
			}
			backend = result.backend ?? backend;
			if (result.timing) {
				elapsedSec += result.timing.elapsedSec;
				audioSec += result.timing.audioSec;
				timedChunks++;
			} else {
				untimedChunks++;
			}
			const runRtf = audioSec > 0 ? elapsedSec / audioSec : undefined;
			// Through `console.*` on purpose: that is what the main-process log file
			// wraps (electron/diagnostics/main-log-file.ts), so these lines
			// reach "Save Diagnostics". The helper's own stderr is forwarded with
			// `process.stderr.write` in whisperServer.ts and never lands there.
			console.info(
				`[stt] chunk ${index + 1}/${chunks.length} on ${backend}: ` +
					(result.timing
						? formatTiming(result.timing)
						: "no timing reported (staged helper binary pre-dates the field)"),
			);
			// Falling back to CPU is silent on both sides — the helper retries
			// without GPU when init returns null, and this process can relaunch the
			// helper with `--cpu` — and it costs about half the throughput. A user
			// staring at a transcription that takes twice as long has, without this,
			// nothing anywhere telling them why.
			if (!warnedCpu && backend === "whispercpp-cpu") {
				warnedCpu = true;
				console.warn(
					"[stt] running on CPU - no GPU backend was bound. Expect roughly half " +
						"the throughput of the Vulkan/Metal path (median 2.07x on the reference " +
						"machine, tools/stt-eval/whispercpp-dtw-poc/REPORT.md 5.3).",
				);
			}
			this.emit({
				phase: "transcribe",
				completedSec: chunk.endSample / SAMPLE_RATE,
				totalSec,
				backend,
				rtf: runRtf,
			});
		}

		// A total has to cover everything it claims to total. Let one chunk go
		// unmeasured and these sums describe a SHORTER recording than the one that
		// was actually transcribed — under a field documented as covering every
		// chunk. So an incomplete run reports no timing rather than partial timing.
		//
		// The per-chunk `rtf` emitted above is deliberately not held to this: it is
		// a RATIO, not a total, and stays honest over whatever subset reported.
		const timing: SttTiming | undefined =
			timedChunks > 0 && untimedChunks === 0 && audioSec > 0
				? { elapsedSec, audioSec, rtf: elapsedSec / audioSec }
				: undefined;
		console.info(
			`[stt] done on ${backend}: ${chunks.length} chunk(s), ` +
				(timing
					? formatTiming(timing)
					: untimedChunks === chunks.length
						? "no timing reported"
						: `timing incomplete (${untimedChunks}/${chunks.length} chunks unmeasured)`),
		);
		return {
			segments,
			wordSegments,
			detectedLanguage: detectedLanguage ?? language ?? "auto",
			backend,
			timing,
		};
	}

	/** Best-effort shutdown; safe to call from `before-quit` hooks. */
	async shutdown(): Promise<void> {
		if (this.shuttingDown) return;
		this.shuttingDown = true;
		this.cancel();
		await this.server.shutdown();
	}
}

let singleton: SttManager | null = null;
let sttShuttingDown = false;

/** Lazy singleton for the IPC layer; processes one transcription at a time. */
export function getSttManager(): SttManager {
	if (sttShuttingDown) throw cancelledError();
	if (!singleton) singleton = new SttManager();
	return singleton;
}

/**
 * Stop and release the lazy singleton without creating one just to quit.
 *
 * Electron's GUI lifecycle awaits this from a guarded `before-quit` handler;
 * clearing the slot first also makes repeated quit events idempotent.
 */
export async function shutdownStt(): Promise<void> {
	sttShuttingDown = true;
	const manager = singleton;
	await manager?.shutdown();
	singleton = null;
}

/** Reset the singleton — for tests. */
export function _resetSttManagerForTests(): void {
	singleton = null;
	sttShuttingDown = false;
}

/**
 * Wire the IPC channels. Call this from `registerIpcHandlers` so the renderer
 * can `invoke("stt:transcribe", request)` and receive `SttTranscribeResponse`,
 * and `invoke("stt:cancel")` to stop a run it no longer wants. Status events
 * fan out on `"stt:status"` (main → renderer push), scoped to the calling
 * `webContents` so two windows don't cross-talk.
 */
export function registerSttIpc(ipcMain: IpcMain): void {
	const manager = getSttManager();
	ipcMain.handle(
		"stt:transcribe",
		async (event, req: SttTranscribeRequest): Promise<SttTranscribeResponse> => {
			const senderId = event.sender.id;
			// Attach for the life of THIS request only. Overlapping requests each
			// own their own sink, so neither can silence the other on the way out.
			const detach = manager.addStatusSink((statusEvent) => {
				if (event.sender.id === senderId && !event.sender.isDestroyed()) {
					event.sender.send("stt:status", statusEvent);
				}
			});
			try {
				return await manager.transcribe(req);
			} finally {
				detach();
			}
		},
	);
	ipcMain.handle("stt:cancel", () => {
		manager.cancel();
	});
}