Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): never hold a chunk on the word aligner's download
EtienneLescotf2051caea3fe
diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.tsindex 99238205f..91cc15d4c 100644--- a/electron/stt/ctcAlign.test.ts+++ b/electron/stt/ctcAlign.test.ts@@ -174,6 +174,17 @@ describe("alignWordsOnEmissions", () => {expect(out[1].endSec).toBe(out[0].endSec);});+ it("aligns a stretch that runs to the end of the upload, past the last frame", () => {+ // 1 s of audio gives 49 frames (0.98 s): the stretch ends after them.+ const tokens = [...gap(30), "a", "b", ...gap(17)];+ const out = alignWordsOnEmissions(+ [word("ab", 0.65, 1, 0.7)],+ [{ startSec: 0.5, endSec: 1 }],+ emissions(tokens),+ );+ expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC));+ });+it("keeps the helper's times where it cannot align", () => {const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)];// The second word is outside every stretch; the first one's letters do not fit.diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.tsindex f243dfe88..335b6c196 100644--- a/electron/stt/ctcAlign.ts+++ b/electron/stt/ctcAlign.ts@@ -198,8 +198,11 @@ export function alignWordsOnEmissions<W extends AlignableWord>(const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);const owned: number[] = [];for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k);+ // A region's frames stop up to one receptive field (25 ms) short of the audio+ // it was cut from, so a stretch that runs to the end of the upload ends+ // past them; two frames of slack keep it. `hi` below stays on the frames.const region = emissions.regions.find(- (r) => r.startSec <= onset && r.startSec + r.frames * strideSec >= offset,+ (r) => r.startSec <= onset && r.startSec + (r.frames + 2) * strideSec >= offset,);if (!region || owned.length === 0) continue;// The frames of this stretch: its margin, stopping at the neighbours' speech.diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.tsindex 8b6d26c53..c5af87fe9 100644--- a/electron/stt/index.test.ts+++ b/electron/stt/index.test.ts@@ -35,6 +35,8 @@ vi.mock("./whisperServer", () => {vi.mock("./modelManager", () => ({ensureModels: vi.fn(async () => undefined),ensureAligner: vi.fn(async ({ language }: { language: string }) => `/fake/${language}.gguf`),+ cachedAligner: vi.fn(async () => null),+ alignerPath: vi.fn((base: string, language: string): string | null => `${base}/${language}.gguf`),modelPaths: (base: string) => ({whisper: `${base}/whisper-ggml/ggml-small-q8_0.bin`,}),@@ -471,39 +473,108 @@ describe("SttManager", () => {expect(fakeWhisperServer.start).toHaveBeenCalledOnce();});- it("hands the chunks an aligner, tried once per run when its download fails", async () => {- const { ensureAligner } = await import("./modelManager");- const mocked = vi.mocked(ensureAligner);- mocked.mockClear();- mocked.mockRejectedValueOnce(new Error("offline"));- // Each chunk asks for the aligner of the language it detected, as the helper does.- const asked: Array<string | null> = [];- fakeWhisperServer.transcribe.mockImplementation(- async (opts: { alignerFor?: (language: string) => Promise<string | null> }) => {- asked.push((await opts.alignerFor?.("fr")) ?? null);- return {- segments: [],- wordSegments: [],- detectedLanguage: "fr",- backend: "whispercpp-cpu" as const,- };- },- );- const mgr = new SttManager();- await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });+ describe("word aligner", () => {+ /** Chunks report what `alignerFor("fr")` gave them, as the helper would use it. */+ function recordAligner() {+ const asked: Array<string | null> = [];+ fakeWhisperServer.transcribe.mockImplementation(+ async (opts: { alignerFor?: (language: string) => Promise<string | null> }) => {+ asked.push((await opts.alignerFor?.("fr")) ?? null);+ return {+ segments: [],+ wordSegments: [],+ detectedLanguage: "fr",+ backend: "whispercpp-cpu" as const,+ };+ },+ );+ return asked;+ }const long = new Float32Array(16_000 * 400);- await mgr.transcribe({ samples: long, language: "fr" });- // Failed once: the rest of the run keeps whisper's times without retrying.- expect(asked.length).toBeGreaterThan(1);- expect(asked.every((x) => x === null)).toBe(true);- expect(mocked).toHaveBeenCalledOnce();-- // The next run tries again, and a success is kept for the session.- asked.length = 0;- await mgr.transcribe({ samples: long, language: "fr" });- await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" });- expect(asked.every((x) => x === "/fake/fr.gguf")).toBe(true);- expect(mocked).toHaveBeenCalledTimes(2);++ async function mocks() {+ const mm = await import("./modelManager");+ const ensure = vi.mocked(mm.ensureAligner);+ const cached = vi.mocked(mm.cachedAligner);+ ensure.mockReset();+ cached.mockReset();+ cached.mockResolvedValue(null);+ return { ensure, cached };+ }++ it("never holds a chunk on the download, and uses the aligner once it lands", async () => {+ const { ensure } = await mocks();+ let land: (file: string) => void = () => undefined;+ ensure.mockImplementation(() => new Promise((resolve) => (land = resolve)));+ const asked = recordAligner();+ const mgr = new SttManager();+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });++ // The download never finishes during this run: every chunk goes on without it.+ await mgr.transcribe({ samples: long, language: "fr" });+ expect(asked.length).toBeGreaterThan(1);+ expect(asked.every((x) => x === null)).toBe(true);+ expect(ensure).toHaveBeenCalledOnce();++ land("/fake/fr.gguf");+ await new Promise((resolve) => setTimeout(resolve, 0));+ asked.length = 0;+ await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" });+ expect(asked).toEqual(["/fake/fr.gguf"]);+ expect(ensure).toHaveBeenCalledOnce();+ });++ it("uses a verified copy already on disk without downloading", async () => {+ const { ensure, cached } = await mocks();+ cached.mockResolvedValue("/fake/fr.gguf");+ const asked = recordAligner();+ const mgr = new SttManager();+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });+ await mgr.transcribe({ samples: long, language: "fr" });+ expect(asked.every((x) => x === "/fake/fr.gguf")).toBe(true);+ expect(cached).toHaveBeenCalledOnce();+ expect(ensure).not.toHaveBeenCalled();+ });++ it("aborts the download on cancel and on shutdown, then retries on a later run", async () => {+ const { ensure } = await mocks();+ const signals: AbortSignal[] = [];+ ensure.mockImplementation(+ ({ signal }: { signal?: AbortSignal }) =>+ new Promise((_, reject) => {+ if (signal) signals.push(signal);+ signal?.addEventListener("abort", () => reject(signal.reason));+ }),+ );+ recordAligner();+ const mgr = new SttManager();+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });+ await mgr.transcribe({ samples: long, language: "fr" });+ expect(signals).toHaveLength(1);++ mgr.cancel();+ expect(signals[0].aborted).toBe(true);+ await new Promise((resolve) => setTimeout(resolve, 0));++ // Not cached as failed for good: the next transcription starts it again.+ await mgr.transcribe({ samples: long, language: "fr" });+ expect(signals).toHaveLength(2);+ expect(signals[1].aborted).toBe(false);+ await mgr.shutdown();+ expect(signals[1].aborted).toBe(true);+ });++ it("gives a language without an aligner nothing, and downloads nothing", async () => {+ const { ensure } = await mocks();+ const mm = await import("./modelManager");+ vi.mocked(mm.alignerPath).mockReturnValueOnce(null);+ const asked = recordAligner();+ const mgr = new SttManager();+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });+ await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" });+ expect(asked).toEqual([null]);+ expect(ensure).not.toHaveBeenCalled();+ });});it("fans status out to every sink, and detaching one leaves the others", async () => {diff --git a/electron/stt/index.ts b/electron/stt/index.tsindex eae7f8a97..11c760f0b 100644--- a/electron/stt/index.ts+++ b/electron/stt/index.ts@@ -2,7 +2,13 @@ import path from "node:path";import { app, type IpcMain } from "electron";import { planChunks } from "./chunking";import { extractMono16kPcm } from "./extractAudio";-import { ensureAligner, ensureModels, modelPaths } from "./modelManager";+import {+ alignerPath,+ cachedAligner,+ ensureAligner,+ ensureModels,+ modelPaths,+} from "./modelManager";import type {SttPhraseSegment,SttStatusEvent,@@ -106,36 +112,58 @@ export class SttManager {private cancelEpoch = 0;/**- * The word aligner per language (ctcAlign.ts), resolved once per session: its- * file is hashed on the way in, and that is not free at 350 MB. A language- * whose download failed is left out for the rest of the run rather than- * retried per chunk, and tried again on the next one.+ * The word aligner per language (ctcAlign.ts). A chunk never waits for a+ * download: it is an improvement, and a 350 MB fetch inside a chunk would hold+ * the helper's queue for as long as the network takes, past any request+ * ceiling and out of Cancel's reach. So:+ * - a copy already on disk is verified in place the first time a language is+ * asked for in a session (a local read, no network), then reused;+ * - otherwise the download starts in the background and the chunks keep+ * whisper's times until it lands; later chunks and transcriptions use it;+ * - Cancel and quit abort it, and so does a stall (`ensureAligner`). A failed+ * one is not retried for the rest of the run, only on the next transcription.*/- private readonly aligners = new Map<string, Promise<string | null>>();+ private readonly alignersReady = new Map<string, string>();+ private readonly alignersChecked = new Set<string>();+ private readonly alignerDownloads = new Map<string, AbortController>();private readonly alignersFailed = new Set<string>();- private readonly alignerFor = (language: string): Promise<string | null> => {- if (this.alignersFailed.has(language)) return Promise.resolve(null);- let resolved = this.aligners.get(language);- if (!resolved) {- resolved = ensureAligner({- baseDir: this.getModelsDir(),- language,- onProgress: (downloadedBytes, totalBytes) =>- this.emit({ phase: "model", model: "ctc-aligner", downloadedBytes, totalBytes }),- }).catch((error: unknown) => {+ private readonly alignerFor = async (language: string): Promise<string | null> => {+ const ready = this.alignersReady.get(language);+ if (ready) return ready;+ if (!alignerPath(this.getModelsDir(), language)) return null;+ if (!this.alignersChecked.has(language)) {+ this.alignersChecked.add(language);+ const cached = await cachedAligner(this.getModelsDir(), language);+ if (cached) {+ this.alignersReady.set(language, cached);+ return cached;+ }+ }+ if (!this.alignersFailed.has(language) && !this.alignerDownloads.has(language)) {+ this.downloadAligner(language);+ }+ return null;+ };++ private downloadAligner(language: string): void {+ const controller = new AbortController();+ this.alignerDownloads.set(language, controller);+ console.info(`[stt] downloading the "${language}" word aligner in the background`);+ ensureAligner({ baseDir: this.getModelsDir(), language, signal: controller.signal })+ .then((file) => {+ if (file) this.alignersReady.set(language, file);+ console.info(`[stt] "${language}" word aligner ready`);+ })+ .catch((error: unknown) => {+ this.alignersFailed.add(language);console.warn(`[stt] no word aligner for "${language}", keeping whisper's word times: ` +`${error instanceof Error ? error.message : String(error)}`,);- this.alignersFailed.add(language);- this.aligners.delete(language);- return null;- });- this.aligners.set(language, resolved);- }- return resolved;- };+ })+ .finally(() => this.alignerDownloads.delete(language));+ }/*** The extraction in flight, if any. `cancelEpoch` alone stops the CHUNK loop, which is@@ -176,6 +204,7 @@ export class SttManager {cancel(): void {this.cancelEpoch++;this.extraction?.abort();+ for (const download of this.alignerDownloads.values()) download.abort(cancelledError());}/**@@ -470,7 +499,7 @@ export class SttManager {async shutdown(): Promise<void> {if (this.shuttingDown) return;this.shuttingDown = true;- this.cancelEpoch++;+ this.cancel();await this.server.shutdown();}}diff --git a/electron/stt/modelManager.test.ts b/electron/stt/modelManager.test.tsindex 0676b1a83..dcee6a24f 100644--- a/electron/stt/modelManager.test.ts+++ b/electron/stt/modelManager.test.ts@@ -8,6 +8,7 @@ import {alignerPath,areModelsPresent,CTC_ALIGNERS,+ cachedAligner,ensureAligner,ensureModels,modelPaths,@@ -283,17 +284,14 @@ describe("modelManager", () => {const bytes = Buffer.from("gguf weights");const original = CTC_ALIGNERS.fr.expectedSha256;CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex");- const progress: number[] = [];try {const file = await ensureAligner({baseDir: dir,language: "fr",fetcher: async () => new Response(bytes, { status: 200 }),- onProgress: (done) => progress.push(done),});expect(file).toBe(alignerPath(dir, "fr"));expect(await readFile(file as string)).toEqual(bytes);- expect(progress[progress.length - 1]).toBe(bytes.length);} finally {CTC_ALIGNERS.fr.expectedSha256 = original;}@@ -309,5 +307,72 @@ describe("modelManager", () => {).rejects.toThrow(/SHA-256 mismatch/);expect(existsSync(alignerPath(dir, "en") as string)).toBe(false);});++ /** A body that sends a few bytes, then nothing, ever. */+ const stalledBody = () =>+ new Response(+ new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new Uint8Array(8));+ },+ }),+ { status: 200 },+ );++ it("aborts a download that stops sending, and leaves no partial file", async () => {+ const file = alignerPath(dir, "fr") as string;+ await expect(+ ensureAligner({+ baseDir: dir,+ language: "fr",+ stallMs: 50,+ fetcher: async () => stalledBody(),+ }),+ ).rejects.toThrow(/stalled/);+ expect(existsSync(file)).toBe(false);+ expect(existsSync(`${file}.partial`)).toBe(false);+ });++ it("stops when its signal aborts, mid-body or between attempts", async () => {+ const controller = new AbortController();+ const download = ensureAligner({+ baseDir: dir,+ language: "fr",+ signal: controller.signal,+ fetcher: async () => stalledBody(),+ });+ setTimeout(() => controller.abort(new Error("cancelled")), 20);+ await expect(download).rejects.toThrow("cancelled");++ // A 503 puts it in a backoff of seconds: the abort must not wait for it.+ const again = new AbortController();+ const started = Date.now();+ const retrying = ensureAligner({+ baseDir: dir,+ language: "fr",+ signal: again.signal,+ fetcher: async () => new Response("busy", { status: 503 }),+ });+ setTimeout(() => again.abort(new Error("quit")), 20);+ await expect(retrying).rejects.toThrow("quit");+ expect(Date.now() - started).toBeLessThan(1000);+ });++ it("verifies a cached copy without the network", async () => {+ expect(await cachedAligner(dir, "fr")).toBeNull();+ const file = alignerPath(dir, "fr") as string;+ const bytes = Buffer.from("gguf weights");+ await mkdir(path.dirname(file), { recursive: true });+ await writeFile(file, bytes);+ // Present but not the pinned bytes: not usable.+ expect(await cachedAligner(dir, "fr")).toBeNull();+ const original = CTC_ALIGNERS.fr.expectedSha256;+ CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex");+ try {+ expect(await cachedAligner(dir, "fr")).toBe(file);+ } finally {+ CTC_ALIGNERS.fr.expectedSha256 = original;+ }+ });});});diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.tsindex 05d6c5c8e..d9d4aef32 100644--- a/electron/stt/modelManager.ts+++ b/electron/stt/modelManager.ts@@ -140,23 +140,41 @@ export function alignerPath(baseDir: string, language: string): string | null {return file ? path.join(baseDir, ALIGNER_DIR, file.name) : null;}+/**+ * The aligner for `language` when it is already on disk and intact, without+ * touching the network: null when it is missing, corrupt, or the language has+ * none. A local read, so a transcription can afford to wait for it.+ */+export async function cachedAligner(baseDir: string, language: string): Promise<string | null> {+ const filePath = alignerPath(baseDir, language);+ if (!filePath || !existsSync(filePath)) return null;+ const actual = await sha256OfFile(filePath).catch(() => "");+ return actual === CTC_ALIGNERS[language].expectedSha256 ? filePath : null;+}++/** Abort an aligner download that receives no bytes for this long. */+export const ALIGNER_STALL_MS = 30_000;+/*** Make sure the aligner for `language` is on disk, downloading and verifying it* like the whisper model. Null when the language has none; throws when the- * download fails, which the caller turns into "keep whisper's times".+ * download fails, stalls for `stallMs` or is aborted through `signal`, which the+ * caller turns into "keep whisper's times".*/export async function ensureAligner(opts: {baseDir: string;language: string;- onProgress?: (downloadedBytes: number, totalBytes: number) => void;+ signal?: AbortSignal;+ stallMs?: number;fetcher?: typeof fetch;}): Promise<string | null> {const filePath = alignerPath(opts.baseDir, opts.language);if (!filePath) return null;const file = CTC_ALIGNERS[opts.language];await ensureFile(filePath, file.url, file.expectedSha256, {- onProgress: (bytes) => opts.onProgress?.(bytes, file.approximateBytes),fetcher: opts.fetcher,+ signal: opts.signal,+ stallMs: opts.stallMs ?? ALIGNER_STALL_MS,});return filePath;}@@ -199,8 +217,19 @@ export async function sha256OfFile(filePath: string): Promise<string> {const MAX_ATTEMPTS = 6;const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);-function sleep(ms: number): Promise<void> {- return new Promise((resolve) => setTimeout(resolve, ms));+function sleep(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise((resolve, reject) => {+ if (signal?.aborted) return reject(signal.reason);+ const timer = setTimeout(resolve, ms);+ signal?.addEventListener(+ "abort",+ () => {+ clearTimeout(timer);+ reject(signal.reason);+ },+ { once: true },+ );+ });}function backoffMs(attempt: number, retryAfter: string | null): number {@@ -213,19 +242,34 @@ function backoffMs(attempt: number, retryAfter: string | null): number {return Math.min(60_000, 2_000 * 2 ** (attempt - 1)) + Math.floor(Math.random() * 1000);}-async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise<Response> {+/**+ * `signal` aborts the request and the waits between attempts; `waiting` is told+ * when a backoff starts and ends, so a stall timer does not count it.+ */+async function fetchWithRetry(+ url: string,+ fetcher: typeof fetch,+ signal?: AbortSignal,+ waiting: (yes: boolean) => void = () => undefined,+): Promise<Response> {let lastErr: unknown;+ const backoff = async (ms: number) => {+ waiting(true);+ await sleep(ms, signal);+ waiting(false);+ };for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {try {const res = await fetcher(url, {headers: { "user-agent": "openscreen-stt" },+ signal,});if (res.ok && res.body) return res;if (res.status >= 400 && res.status < 500 && !RETRYABLE_STATUS.has(res.status)) {throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`);}if (RETRYABLE_STATUS.has(res.status) && attempt < MAX_ATTEMPTS) {- await sleep(backoffMs(attempt, res.headers.get("retry-after")));+ await backoff(backoffMs(attempt, res.headers.get("retry-after")));continue;}throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`);@@ -234,8 +278,8 @@ async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise<Respoif (err instanceof Error && err.message.startsWith("Failed to download")) {throw err;}- if (attempt >= MAX_ATTEMPTS) throw err;- await sleep(backoffMs(attempt, null));+ if (signal?.aborted || attempt >= MAX_ATTEMPTS) throw signal?.reason ?? err;+ await backoff(backoffMs(attempt, null));}}throw lastErr;@@ -246,6 +290,10 @@ export interface DownloadOptions {onProgress?: (bytes: number) => void;/** Override fetch (for tests); defaults to `globalThis.fetch`. */fetcher?: typeof fetch;+ /** Aborts the download (and the waits between attempts). */+ signal?: AbortSignal;+ /** Abort when no byte arrives for this long, backoffs aside. Off when unset. */+ stallMs?: number;}/**@@ -279,17 +327,44 @@ async function ensureFile(await mkdir(path.dirname(filePath), { recursive: true });const fetcher = options.fetcher ?? fetch;- const res = await fetchWithRetry(fileUrl, fetcher);const tmp = `${filePath}.partial`;- let downloaded = 0;-- const source = Readable.fromWeb(res.body as never);- source.on("data", (chunk: Buffer | Uint8Array) => {- downloaded += chunk.length;- options.onProgress?.(downloaded);- });- const { createWriteStream } = await import("node:fs");- await pipeline(source, createWriteStream(tmp));+ // One controller for the caller's abort and the stall timer: a connection that+ // stops sending leaves `pipeline` pending forever, with nothing else to end it.+ const controller = new AbortController();+ const onAbort = () => controller.abort(options.signal?.reason);+ if (options.signal?.aborted) onAbort();+ options.signal?.addEventListener("abort", onAbort, { once: true });+ let stall: ReturnType<typeof setTimeout> | undefined;+ const arm = (on = true) => {+ clearTimeout(stall);+ if (!on || !options.stallMs) return;+ const ms = options.stallMs;+ stall = setTimeout(+ () => controller.abort(new Error(`download stalled: no data for ${ms / 1000}s`)),+ ms,+ );+ };+ try {+ arm();+ const res = await fetchWithRetry(fileUrl, fetcher, controller.signal, (waiting) =>+ arm(!waiting),+ );+ let downloaded = 0;+ const source = Readable.fromWeb(res.body as never);+ source.on("data", (chunk: Buffer | Uint8Array) => {+ arm();+ downloaded += chunk.length;+ options.onProgress?.(downloaded);+ });+ const { createWriteStream } = await import("node:fs");+ await pipeline(source, createWriteStream(tmp), { signal: controller.signal });+ } catch (error) {+ await rm(tmp, { force: true }).catch(() => undefined);+ throw controller.signal.aborted ? (controller.signal.reason ?? error) : error;+ } finally {+ arm(false);+ options.signal?.removeEventListener("abort", onAbort);+ }if (expectedSha256) {const actual = await sha256OfFile(tmp);diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.tsindex 4815da53b..5433698ac 100644--- a/electron/stt/transcriptionContract.ts+++ b/electron/stt/transcriptionContract.ts@@ -71,7 +71,7 @@ export interface SttStatusEvent {/** Total bytes for the in-flight download. */totalBytes?: number;/** Which model is downloading. */- model?: "whisper" | "silero-vad" | "ctc-aligner";+ model?: "whisper" | "silero-vad";/*** Seconds of audio transcribed so far, and the total for this request. Only* when `phase === "transcribe"`. Progress is reported per CHUNK (seediff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.tsindex bea7c25b5..196c9c011 100644--- a/electron/stt/whisperServer.ts+++ b/electron/stt/whisperServer.ts@@ -585,7 +585,8 @@ export class WhisperServerManager {* Run one transcription; serializes concurrent callers.** `alignerFor` resolves the CTC aligner model for the language whisper- * detected (downloading it if need be), or null when there is none. With one,+ * detected, or null when there is none or it is not ready yet (it must not+ * wait on the network: no download runs inside a chunk, see SttManager). With one,* the words are re-timed on it before their phrase edges go on the speech* (ctcAlign.ts); any failure there keeps whisper's own times.*/diff --git a/scripts/test-whisper-stt.mjs b/scripts/test-whisper-stt.mjsindex 470b40793..8ea9f2a34 100644--- a/scripts/test-whisper-stt.mjs+++ b/scripts/test-whisper-stt.mjs@@ -254,6 +254,7 @@ async function main() {let json;let emissions = null;let alignerModel = null;+ let alignerSkipped = "";try {await waitForReady(`http://127.0.0.1:${port}/`);const form = new FormData();@@ -284,7 +285,13 @@ async function main() {if (!ares.ok) throw new Error(`/emissions returned ${ares.status}: ${await ares.text()}`);emissions = await ares.json();} else {- alignerModel = null;+ // Say which file was looked for: a custom OPENSCREEN_WHISPER_MODEL moves+ // the cache this path is derived from, and would read as "no aligner".+ alignerSkipped = !WITH_VAD+ ? "no VAD model, so no speech to align"+ : alignerModel+ ? `no aligner at ${alignerModel} (set OPENSCREEN_ALIGNER_MODEL)`+ : `no aligner for "${json.detected_language}"`;}} finally {child.kill();@@ -458,7 +465,7 @@ async function main() {console.log(`aligner: ${emissions.elapsed_s.toFixed(2)}s on ${emissions.device}`);}} else {- console.log("\n(no aligner for this language in the cache: aligner checks skipped)");+ console.log(`\n(aligner checks skipped: ${alignerSkipped})`);}if (json.timing) {diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.mdindex 35b9187af..2467004e6 100644--- a/technical-documentation/architecture/transcription-and-captions.md+++ b/technical-documentation/architecture/transcription-and-captions.md@@ -112,10 +112,9 @@ renderer imposes a timeout that a slow download could trip: the preload does abare `ipcRenderer.invoke`, `fetchWithRetry` has no per-request deadline, andwhisper-server's 30 s readiness budget only starts once the download resolved.-The word aligner (§ Word-level alignment, step 5) is the exception to "first-run only": it is fetched the first time a transcription detects English or-French (109 or 348 MB), in the middle of that run, which then simply takes-longer once more.+The word aligner (§ Word-level alignment, step 5) is not part of that phase: it+is fetched in the background the first time a transcription detects English or+French (109 or 348 MB), and no chunk ever waits for it.Three edges make that promise hold, and each is load-bearing:@@ -288,9 +287,9 @@ it verbatim in the response.midpoint. Next to a pause (a gap of 100 ms or more between letters) theedges move further out, 60 ms before and 150 ms after, both capped atmid-pause: early there is silence, late is an audible attack.- - **Fallback.** No aligner for the language, a download that failed, a- helper without `/emissions`, a stretch whose letters do not fit in its- frames: the words keep the times of step 4. Step 6 runs either way.+ - **Fallback.** No aligner for the language, a download not landed yet or+ failed, a helper without `/emissions`, a stretch whose letters do not fit in+ its frames: the words keep the times of step 4. Step 6 runs either way.6. **Anchor phrase edges on the speech** —[`electron/stt/snapWordBoundaries.ts`](../../electron/stt/snapWordBoundaries.ts)leaves boundaries inside a phrase alone: they are within ~16 ms of the audio@@ -390,10 +389,13 @@ digest an invariant. Bumping the model therefore means bumping the revisionand the digest together.**The word aligners** (step 5) are fetched only for a language that has one,-the first time a transcription detects it, inside the same run (`phase:-"model"`, `model: "ctc-aligner"`), into `stt-models/ctc-aligner/`. A failed-download keeps whisper's times for the rest of the run and is retried on the-next one.+into `stt-models/ctc-aligner/`, and never inside a chunk. The first time a+session meets a language, a copy already on disk is verified in place (a local+read). Without one, the download starts in the background+(`SttManager.alignerFor`): the chunks that run before it lands keep whisper's+times, the later ones and later transcriptions use it. A download stalled for+30 s, cancelled with the transcription, or interrupted by quitting is+abandoned, and retried on the next transcription rather than per chunk.| Language | Model | Download ||---|---|---|@@ -463,7 +465,11 @@ timings present, the DTW guardrail passed, `backend` reporting GPU offload on aGPU-capable host, `detected_language` resolved rather than echoed, word timesmonotonic and inside the clip, and WER against a reference. On macOS itsynthesizes its own clip with `say`, so it needs no fixture; elsewhere pass-`--wav <file>` (and optionally `--ref "<expected text>"`). This is the check+`--wav <file>` (and optionally `--ref "<expected text>"`). When the detected+language's aligner is cached (or `OPENSCREEN_ALIGNER_MODEL` names one), it also+calls `/emissions` and checks that the answer is well-formed log-probabilities,+computed on the GPU on a GPU-capable host, decoding to roughly what whisper+heard (letter error rate), with the aligned words still ordered. This is the checkthat the unit tests structurally cannot make: they mock `fetch`, so they assertagainst a hand-written fixture rather than the binary.diff --git a/tools/stt-eval/word-timing/evaluate.mjs b/tools/stt-eval/word-timing/evaluate.mjsindex 4c7cbe873..8a08b718f 100644--- a/tools/stt-eval/word-timing/evaluate.mjs+++ b/tools/stt-eval/word-timing/evaluate.mjs@@ -126,9 +126,12 @@ for (const file of readdirSync(rawDir)const full = postPass(rawWords, samples, speech);const tsMs = performance.now() - t0;const variants = { raw: rawWords, post, "post+vad": full };- if (json.emissions && speech) {+ const emissions = json.emissions ? ctc.parseEmissions(json.emissions) : null;+ if (json.emissions && !emissions)+ console.warn(`${file}: unreadable emissions, CTC stages skipped`);+ if (emissions && speech) {const t1 = performance.now();- const aligned = ctc.alignWordsOnEmissions(rawWords, speech, ctc.parseEmissions(json.emissions));+ const aligned = ctc.alignWordsOnEmissions(rawWords, speech, emissions);variants.ctc = aligned;variants["ctc+vad"] = postPass(aligned, samples, speech);ctcMs += performance.now() - t1;diff --git a/tools/stt-eval/word-timing/real-check.mjs b/tools/stt-eval/word-timing/real-check.mjsindex b0fca0230..a0d9ea90c 100644--- a/tools/stt-eval/word-timing/real-check.mjs+++ b/tools/stt-eval/word-timing/real-check.mjs@@ -66,8 +66,11 @@ const raw = json.segments.filter((w) => w.word);const speech = json.speech.map((s) => ({ startSec: s.start, endSec: s.end }));const post = anchorWordsOnSpeech(raw, speech);-const aligned = emissions- ? anchorWordsOnSpeech(alignWordsOnEmissions(raw, speech, parseEmissions(emissions)), speech)+const parsed = emissions ? parseEmissions(emissions) : null;+if (emissions && !parsed)+ throw new Error("/emissions answered something that is not letter scores (vocab/regions/sizes)");+const aligned = parsed+ ? anchorWordsOnSpeech(alignWordsOnEmissions(raw, speech, parsed), speech): null;mkdirSync(RESULTS, { recursive: true });writeFileSync(