Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(stt): never hold a chunk on the word aligner's download

EtienneLescotf2051caea3fe
diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.ts
index 99238205f..91cc15d4c 100644
--- a/electron/stt/ctcAlign.test.ts
+++ b/electron/stt/ctcAlign.test.ts
@@ -174,6 +174,17 @@ describe("alignWordsOnEmissions", () => {
expect(out[1].endSec).toBe(out[0].endSec);
});
+ it("aligns a stretch that runs to the end of the upload, past the last frame", () => {
+ // 1 s of audio gives 49 frames (0.98 s): the stretch ends after them.
+ const tokens = [...gap(30), "a", "b", ...gap(17)];
+ const out = alignWordsOnEmissions(
+ [word("ab", 0.65, 1, 0.7)],
+ [{ startSec: 0.5, endSec: 1 }],
+ emissions(tokens),
+ );
+ expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC));
+ });
+
it("keeps the helper's times where it cannot align", () => {
const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)];
// The second word is outside every stretch; the first one's letters do not fit.
diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.ts
index f243dfe88..335b6c196 100644
--- a/electron/stt/ctcAlign.ts
+++ b/electron/stt/ctcAlign.ts
@@ -198,8 +198,11 @@ export function alignWordsOnEmissions<W extends AlignableWord>(
const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
const owned: number[] = [];
for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k);
+ // A region's frames stop up to one receptive field (25 ms) short of the audio
+ // it was cut from, so a stretch that runs to the end of the upload ends
+ // past them; two frames of slack keep it. `hi` below stays on the frames.
const region = emissions.regions.find(
- (r) => r.startSec <= onset && r.startSec + r.frames * strideSec >= offset,
+ (r) => r.startSec <= onset && r.startSec + (r.frames + 2) * strideSec >= offset,
);
if (!region || owned.length === 0) continue;
// The frames of this stretch: its margin, stopping at the neighbours' speech.
diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts
index 8b6d26c53..c5af87fe9 100644
--- a/electron/stt/index.test.ts
+++ b/electron/stt/index.test.ts
@@ -35,6 +35,8 @@ vi.mock("./whisperServer", () => {
vi.mock("./modelManager", () => ({
ensureModels: vi.fn(async () => undefined),
ensureAligner: vi.fn(async ({ language }: { language: string }) => `/fake/${language}.gguf`),
+ cachedAligner: vi.fn(async () => null),
+ alignerPath: vi.fn((base: string, language: string): string | null => `${base}/${language}.gguf`),
modelPaths: (base: string) => ({
whisper: `${base}/whisper-ggml/ggml-small-q8_0.bin`,
}),
@@ -471,39 +473,108 @@ describe("SttManager", () => {
expect(fakeWhisperServer.start).toHaveBeenCalledOnce();
});
- it("hands the chunks an aligner, tried once per run when its download fails", async () => {
- const { ensureAligner } = await import("./modelManager");
- const mocked = vi.mocked(ensureAligner);
- mocked.mockClear();
- mocked.mockRejectedValueOnce(new Error("offline"));
- // Each chunk asks for the aligner of the language it detected, as the helper does.
- const asked: Array<string | null> = [];
- fakeWhisperServer.transcribe.mockImplementation(
- async (opts: { alignerFor?: (language: string) => Promise<string | null> }) => {
- asked.push((await opts.alignerFor?.("fr")) ?? null);
- return {
- segments: [],
- wordSegments: [],
- detectedLanguage: "fr",
- backend: "whispercpp-cpu" as const,
- };
- },
- );
- const mgr = new SttManager();
- await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });
+ describe("word aligner", () => {
+ /** Chunks report what `alignerFor("fr")` gave them, as the helper would use it. */
+ function recordAligner() {
+ const asked: Array<string | null> = [];
+ fakeWhisperServer.transcribe.mockImplementation(
+ async (opts: { alignerFor?: (language: string) => Promise<string | null> }) => {
+ asked.push((await opts.alignerFor?.("fr")) ?? null);
+ return {
+ segments: [],
+ wordSegments: [],
+ detectedLanguage: "fr",
+ backend: "whispercpp-cpu" as const,
+ };
+ },
+ );
+ return asked;
+ }
const long = new Float32Array(16_000 * 400);
- await mgr.transcribe({ samples: long, language: "fr" });
- // Failed once: the rest of the run keeps whisper's times without retrying.
- expect(asked.length).toBeGreaterThan(1);
- expect(asked.every((x) => x === null)).toBe(true);
- expect(mocked).toHaveBeenCalledOnce();
-
- // The next run tries again, and a success is kept for the session.
- asked.length = 0;
- await mgr.transcribe({ samples: long, language: "fr" });
- await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" });
- expect(asked.every((x) => x === "/fake/fr.gguf")).toBe(true);
- expect(mocked).toHaveBeenCalledTimes(2);
+
+ async function mocks() {
+ const mm = await import("./modelManager");
+ const ensure = vi.mocked(mm.ensureAligner);
+ const cached = vi.mocked(mm.cachedAligner);
+ ensure.mockReset();
+ cached.mockReset();
+ cached.mockResolvedValue(null);
+ return { ensure, cached };
+ }
+
+ it("never holds a chunk on the download, and uses the aligner once it lands", async () => {
+ const { ensure } = await mocks();
+ let land: (file: string) => void = () => undefined;
+ ensure.mockImplementation(() => new Promise((resolve) => (land = resolve)));
+ const asked = recordAligner();
+ const mgr = new SttManager();
+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });
+
+ // The download never finishes during this run: every chunk goes on without it.
+ await mgr.transcribe({ samples: long, language: "fr" });
+ expect(asked.length).toBeGreaterThan(1);
+ expect(asked.every((x) => x === null)).toBe(true);
+ expect(ensure).toHaveBeenCalledOnce();
+
+ land("/fake/fr.gguf");
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ asked.length = 0;
+ await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" });
+ expect(asked).toEqual(["/fake/fr.gguf"]);
+ expect(ensure).toHaveBeenCalledOnce();
+ });
+
+ it("uses a verified copy already on disk without downloading", async () => {
+ const { ensure, cached } = await mocks();
+ cached.mockResolvedValue("/fake/fr.gguf");
+ const asked = recordAligner();
+ const mgr = new SttManager();
+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });
+ await mgr.transcribe({ samples: long, language: "fr" });
+ expect(asked.every((x) => x === "/fake/fr.gguf")).toBe(true);
+ expect(cached).toHaveBeenCalledOnce();
+ expect(ensure).not.toHaveBeenCalled();
+ });
+
+ it("aborts the download on cancel and on shutdown, then retries on a later run", async () => {
+ const { ensure } = await mocks();
+ const signals: AbortSignal[] = [];
+ ensure.mockImplementation(
+ ({ signal }: { signal?: AbortSignal }) =>
+ new Promise((_, reject) => {
+ if (signal) signals.push(signal);
+ signal?.addEventListener("abort", () => reject(signal.reason));
+ }),
+ );
+ recordAligner();
+ const mgr = new SttManager();
+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });
+ await mgr.transcribe({ samples: long, language: "fr" });
+ expect(signals).toHaveLength(1);
+
+ mgr.cancel();
+ expect(signals[0].aborted).toBe(true);
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ // Not cached as failed for good: the next transcription starts it again.
+ await mgr.transcribe({ samples: long, language: "fr" });
+ expect(signals).toHaveLength(2);
+ expect(signals[1].aborted).toBe(false);
+ await mgr.shutdown();
+ expect(signals[1].aborted).toBe(true);
+ });
+
+ it("gives a language without an aligner nothing, and downloads nothing", async () => {
+ const { ensure } = await mocks();
+ const mm = await import("./modelManager");
+ vi.mocked(mm.alignerPath).mockReturnValueOnce(null);
+ const asked = recordAligner();
+ const mgr = new SttManager();
+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });
+ await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" });
+ expect(asked).toEqual([null]);
+ expect(ensure).not.toHaveBeenCalled();
+ });
});
it("fans status out to every sink, and detaching one leaves the others", async () => {
diff --git a/electron/stt/index.ts b/electron/stt/index.ts
index eae7f8a97..11c760f0b 100644
--- a/electron/stt/index.ts
+++ b/electron/stt/index.ts
@@ -2,7 +2,13 @@ import path from "node:path";
import { app, type IpcMain } from "electron";
import { planChunks } from "./chunking";
import { extractMono16kPcm } from "./extractAudio";
-import { ensureAligner, ensureModels, modelPaths } from "./modelManager";
+import {
+ alignerPath,
+ cachedAligner,
+ ensureAligner,
+ ensureModels,
+ modelPaths,
+} from "./modelManager";
import type {
SttPhraseSegment,
SttStatusEvent,
@@ -106,36 +112,58 @@ export class SttManager {
private cancelEpoch = 0;
/**
- * The word aligner per language (ctcAlign.ts), resolved once per session: its
- * file is hashed on the way in, and that is not free at 350 MB. A language
- * whose download failed is left out for the rest of the run rather than
- * retried per chunk, and tried again on the next one.
+ * The word aligner per language (ctcAlign.ts). A chunk never waits for a
+ * download: it is an improvement, and a 350 MB fetch inside a chunk would hold
+ * the helper's queue for as long as the network takes, past any request
+ * ceiling and out of Cancel's reach. So:
+ * - a copy already on disk is verified in place the first time a language is
+ * asked for in a session (a local read, no network), then reused;
+ * - otherwise the download starts in the background and the chunks keep
+ * whisper's times until it lands; later chunks and transcriptions use it;
+ * - Cancel and quit abort it, and so does a stall (`ensureAligner`). A failed
+ * one is not retried for the rest of the run, only on the next transcription.
*/
- private readonly aligners = new Map<string, Promise<string | null>>();
+ private readonly alignersReady = new Map<string, string>();
+ private readonly alignersChecked = new Set<string>();
+ private readonly alignerDownloads = new Map<string, AbortController>();
private readonly alignersFailed = new Set<string>();
- private readonly alignerFor = (language: string): Promise<string | null> => {
- if (this.alignersFailed.has(language)) return Promise.resolve(null);
- let resolved = this.aligners.get(language);
- if (!resolved) {
- resolved = ensureAligner({
- baseDir: this.getModelsDir(),
- language,
- onProgress: (downloadedBytes, totalBytes) =>
- this.emit({ phase: "model", model: "ctc-aligner", downloadedBytes, totalBytes }),
- }).catch((error: unknown) => {
+ private readonly alignerFor = async (language: string): Promise<string | null> => {
+ const ready = this.alignersReady.get(language);
+ if (ready) return ready;
+ if (!alignerPath(this.getModelsDir(), language)) return null;
+ if (!this.alignersChecked.has(language)) {
+ this.alignersChecked.add(language);
+ const cached = await cachedAligner(this.getModelsDir(), language);
+ if (cached) {
+ this.alignersReady.set(language, cached);
+ return cached;
+ }
+ }
+ if (!this.alignersFailed.has(language) && !this.alignerDownloads.has(language)) {
+ this.downloadAligner(language);
+ }
+ return null;
+ };
+
+ private downloadAligner(language: string): void {
+ const controller = new AbortController();
+ this.alignerDownloads.set(language, controller);
+ console.info(`[stt] downloading the "${language}" word aligner in the background`);
+ ensureAligner({ baseDir: this.getModelsDir(), language, signal: controller.signal })
+ .then((file) => {
+ if (file) this.alignersReady.set(language, file);
+ console.info(`[stt] "${language}" word aligner ready`);
+ })
+ .catch((error: unknown) => {
+ this.alignersFailed.add(language);
console.warn(
`[stt] no word aligner for "${language}", keeping whisper's word times: ` +
`${error instanceof Error ? error.message : String(error)}`,
);
- this.alignersFailed.add(language);
- this.aligners.delete(language);
- return null;
- });
- this.aligners.set(language, resolved);
- }
- return resolved;
- };
+ })
+ .finally(() => this.alignerDownloads.delete(language));
+ }
/**
* The extraction in flight, if any. `cancelEpoch` alone stops the CHUNK loop, which is
@@ -176,6 +204,7 @@ export class SttManager {
cancel(): void {
this.cancelEpoch++;
this.extraction?.abort();
+ for (const download of this.alignerDownloads.values()) download.abort(cancelledError());
}
/**
@@ -470,7 +499,7 @@ export class SttManager {
async shutdown(): Promise<void> {
if (this.shuttingDown) return;
this.shuttingDown = true;
- this.cancelEpoch++;
+ this.cancel();
await this.server.shutdown();
}
}
diff --git a/electron/stt/modelManager.test.ts b/electron/stt/modelManager.test.ts
index 0676b1a83..dcee6a24f 100644
--- a/electron/stt/modelManager.test.ts
+++ b/electron/stt/modelManager.test.ts
@@ -8,6 +8,7 @@ import {
alignerPath,
areModelsPresent,
CTC_ALIGNERS,
+ cachedAligner,
ensureAligner,
ensureModels,
modelPaths,
@@ -283,17 +284,14 @@ describe("modelManager", () => {
const bytes = Buffer.from("gguf weights");
const original = CTC_ALIGNERS.fr.expectedSha256;
CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex");
- const progress: number[] = [];
try {
const file = await ensureAligner({
baseDir: dir,
language: "fr",
fetcher: async () => new Response(bytes, { status: 200 }),
- onProgress: (done) => progress.push(done),
});
expect(file).toBe(alignerPath(dir, "fr"));
expect(await readFile(file as string)).toEqual(bytes);
- expect(progress[progress.length - 1]).toBe(bytes.length);
} finally {
CTC_ALIGNERS.fr.expectedSha256 = original;
}
@@ -309,5 +307,72 @@ describe("modelManager", () => {
).rejects.toThrow(/SHA-256 mismatch/);
expect(existsSync(alignerPath(dir, "en") as string)).toBe(false);
});
+
+ /** A body that sends a few bytes, then nothing, ever. */
+ const stalledBody = () =>
+ new Response(
+ new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(new Uint8Array(8));
+ },
+ }),
+ { status: 200 },
+ );
+
+ it("aborts a download that stops sending, and leaves no partial file", async () => {
+ const file = alignerPath(dir, "fr") as string;
+ await expect(
+ ensureAligner({
+ baseDir: dir,
+ language: "fr",
+ stallMs: 50,
+ fetcher: async () => stalledBody(),
+ }),
+ ).rejects.toThrow(/stalled/);
+ expect(existsSync(file)).toBe(false);
+ expect(existsSync(`${file}.partial`)).toBe(false);
+ });
+
+ it("stops when its signal aborts, mid-body or between attempts", async () => {
+ const controller = new AbortController();
+ const download = ensureAligner({
+ baseDir: dir,
+ language: "fr",
+ signal: controller.signal,
+ fetcher: async () => stalledBody(),
+ });
+ setTimeout(() => controller.abort(new Error("cancelled")), 20);
+ await expect(download).rejects.toThrow("cancelled");
+
+ // A 503 puts it in a backoff of seconds: the abort must not wait for it.
+ const again = new AbortController();
+ const started = Date.now();
+ const retrying = ensureAligner({
+ baseDir: dir,
+ language: "fr",
+ signal: again.signal,
+ fetcher: async () => new Response("busy", { status: 503 }),
+ });
+ setTimeout(() => again.abort(new Error("quit")), 20);
+ await expect(retrying).rejects.toThrow("quit");
+ expect(Date.now() - started).toBeLessThan(1000);
+ });
+
+ it("verifies a cached copy without the network", async () => {
+ expect(await cachedAligner(dir, "fr")).toBeNull();
+ const file = alignerPath(dir, "fr") as string;
+ const bytes = Buffer.from("gguf weights");
+ await mkdir(path.dirname(file), { recursive: true });
+ await writeFile(file, bytes);
+ // Present but not the pinned bytes: not usable.
+ expect(await cachedAligner(dir, "fr")).toBeNull();
+ const original = CTC_ALIGNERS.fr.expectedSha256;
+ CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex");
+ try {
+ expect(await cachedAligner(dir, "fr")).toBe(file);
+ } finally {
+ CTC_ALIGNERS.fr.expectedSha256 = original;
+ }
+ });
});
});
diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.ts
index 05d6c5c8e..d9d4aef32 100644
--- a/electron/stt/modelManager.ts
+++ b/electron/stt/modelManager.ts
@@ -140,23 +140,41 @@ export function alignerPath(baseDir: string, language: string): string | null {
return file ? path.join(baseDir, ALIGNER_DIR, file.name) : null;
}
+/**
+ * The aligner for `language` when it is already on disk and intact, without
+ * touching the network: null when it is missing, corrupt, or the language has
+ * none. A local read, so a transcription can afford to wait for it.
+ */
+export async function cachedAligner(baseDir: string, language: string): Promise<string | null> {
+ const filePath = alignerPath(baseDir, language);
+ if (!filePath || !existsSync(filePath)) return null;
+ const actual = await sha256OfFile(filePath).catch(() => "");
+ return actual === CTC_ALIGNERS[language].expectedSha256 ? filePath : null;
+}
+
+/** Abort an aligner download that receives no bytes for this long. */
+export const ALIGNER_STALL_MS = 30_000;
+
/**
* Make sure the aligner for `language` is on disk, downloading and verifying it
* like the whisper model. Null when the language has none; throws when the
- * download fails, which the caller turns into "keep whisper's times".
+ * download fails, stalls for `stallMs` or is aborted through `signal`, which the
+ * caller turns into "keep whisper's times".
*/
export async function ensureAligner(opts: {
baseDir: string;
language: string;
- onProgress?: (downloadedBytes: number, totalBytes: number) => void;
+ signal?: AbortSignal;
+ stallMs?: number;
fetcher?: typeof fetch;
}): Promise<string | null> {
const filePath = alignerPath(opts.baseDir, opts.language);
if (!filePath) return null;
const file = CTC_ALIGNERS[opts.language];
await ensureFile(filePath, file.url, file.expectedSha256, {
- onProgress: (bytes) => opts.onProgress?.(bytes, file.approximateBytes),
fetcher: opts.fetcher,
+ signal: opts.signal,
+ stallMs: opts.stallMs ?? ALIGNER_STALL_MS,
});
return filePath;
}
@@ -199,8 +217,19 @@ export async function sha256OfFile(filePath: string): Promise<string> {
const MAX_ATTEMPTS = 6;
const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
-function sleep(ms: number): Promise<void> {
- return new Promise((resolve) => setTimeout(resolve, ms));
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted) return reject(signal.reason);
+ const timer = setTimeout(resolve, ms);
+ signal?.addEventListener(
+ "abort",
+ () => {
+ clearTimeout(timer);
+ reject(signal.reason);
+ },
+ { once: true },
+ );
+ });
}
function backoffMs(attempt: number, retryAfter: string | null): number {
@@ -213,19 +242,34 @@ function backoffMs(attempt: number, retryAfter: string | null): number {
return Math.min(60_000, 2_000 * 2 ** (attempt - 1)) + Math.floor(Math.random() * 1000);
}
-async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise<Response> {
+/**
+ * `signal` aborts the request and the waits between attempts; `waiting` is told
+ * when a backoff starts and ends, so a stall timer does not count it.
+ */
+async function fetchWithRetry(
+ url: string,
+ fetcher: typeof fetch,
+ signal?: AbortSignal,
+ waiting: (yes: boolean) => void = () => undefined,
+): Promise<Response> {
let lastErr: unknown;
+ const backoff = async (ms: number) => {
+ waiting(true);
+ await sleep(ms, signal);
+ waiting(false);
+ };
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
try {
const res = await fetcher(url, {
headers: { "user-agent": "openscreen-stt" },
+ signal,
});
if (res.ok && res.body) return res;
if (res.status >= 400 && res.status < 500 && !RETRYABLE_STATUS.has(res.status)) {
throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`);
}
if (RETRYABLE_STATUS.has(res.status) && attempt < MAX_ATTEMPTS) {
- await sleep(backoffMs(attempt, res.headers.get("retry-after")));
+ await backoff(backoffMs(attempt, res.headers.get("retry-after")));
continue;
}
throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`);
@@ -234,8 +278,8 @@ async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise<Respo
if (err instanceof Error && err.message.startsWith("Failed to download")) {
throw err;
}
- if (attempt >= MAX_ATTEMPTS) throw err;
- await sleep(backoffMs(attempt, null));
+ if (signal?.aborted || attempt >= MAX_ATTEMPTS) throw signal?.reason ?? err;
+ await backoff(backoffMs(attempt, null));
}
}
throw lastErr;
@@ -246,6 +290,10 @@ export interface DownloadOptions {
onProgress?: (bytes: number) => void;
/** Override fetch (for tests); defaults to `globalThis.fetch`. */
fetcher?: typeof fetch;
+ /** Aborts the download (and the waits between attempts). */
+ signal?: AbortSignal;
+ /** Abort when no byte arrives for this long, backoffs aside. Off when unset. */
+ stallMs?: number;
}
/**
@@ -279,17 +327,44 @@ async function ensureFile(
await mkdir(path.dirname(filePath), { recursive: true });
const fetcher = options.fetcher ?? fetch;
- const res = await fetchWithRetry(fileUrl, fetcher);
const tmp = `${filePath}.partial`;
- let downloaded = 0;
-
- const source = Readable.fromWeb(res.body as never);
- source.on("data", (chunk: Buffer | Uint8Array) => {
- downloaded += chunk.length;
- options.onProgress?.(downloaded);
- });
- const { createWriteStream } = await import("node:fs");
- await pipeline(source, createWriteStream(tmp));
+ // One controller for the caller's abort and the stall timer: a connection that
+ // stops sending leaves `pipeline` pending forever, with nothing else to end it.
+ const controller = new AbortController();
+ const onAbort = () => controller.abort(options.signal?.reason);
+ if (options.signal?.aborted) onAbort();
+ options.signal?.addEventListener("abort", onAbort, { once: true });
+ let stall: ReturnType<typeof setTimeout> | undefined;
+ const arm = (on = true) => {
+ clearTimeout(stall);
+ if (!on || !options.stallMs) return;
+ const ms = options.stallMs;
+ stall = setTimeout(
+ () => controller.abort(new Error(`download stalled: no data for ${ms / 1000}s`)),
+ ms,
+ );
+ };
+ try {
+ arm();
+ const res = await fetchWithRetry(fileUrl, fetcher, controller.signal, (waiting) =>
+ arm(!waiting),
+ );
+ let downloaded = 0;
+ const source = Readable.fromWeb(res.body as never);
+ source.on("data", (chunk: Buffer | Uint8Array) => {
+ arm();
+ downloaded += chunk.length;
+ options.onProgress?.(downloaded);
+ });
+ const { createWriteStream } = await import("node:fs");
+ await pipeline(source, createWriteStream(tmp), { signal: controller.signal });
+ } catch (error) {
+ await rm(tmp, { force: true }).catch(() => undefined);
+ throw controller.signal.aborted ? (controller.signal.reason ?? error) : error;
+ } finally {
+ arm(false);
+ options.signal?.removeEventListener("abort", onAbort);
+ }
if (expectedSha256) {
const actual = await sha256OfFile(tmp);
diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts
index 4815da53b..5433698ac 100644
--- a/electron/stt/transcriptionContract.ts
+++ b/electron/stt/transcriptionContract.ts
@@ -71,7 +71,7 @@ export interface SttStatusEvent {
/** Total bytes for the in-flight download. */
totalBytes?: number;
/** Which model is downloading. */
- model?: "whisper" | "silero-vad" | "ctc-aligner";
+ model?: "whisper" | "silero-vad";
/**
* Seconds of audio transcribed so far, and the total for this request. Only
* when `phase === "transcribe"`. Progress is reported per CHUNK (see
diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts
index bea7c25b5..196c9c011 100644
--- a/electron/stt/whisperServer.ts
+++ b/electron/stt/whisperServer.ts
@@ -585,7 +585,8 @@ export class WhisperServerManager {
* Run one transcription; serializes concurrent callers.
*
* `alignerFor` resolves the CTC aligner model for the language whisper
- * detected (downloading it if need be), or null when there is none. With one,
+ * detected, or null when there is none or it is not ready yet (it must not
+ * wait on the network: no download runs inside a chunk, see SttManager). With one,
* the words are re-timed on it before their phrase edges go on the speech
* (ctcAlign.ts); any failure there keeps whisper's own times.
*/
diff --git a/scripts/test-whisper-stt.mjs b/scripts/test-whisper-stt.mjs
index 470b40793..8ea9f2a34 100644
--- a/scripts/test-whisper-stt.mjs
+++ b/scripts/test-whisper-stt.mjs
@@ -254,6 +254,7 @@ async function main() {
let json;
let emissions = null;
let alignerModel = null;
+ let alignerSkipped = "";
try {
await waitForReady(`http://127.0.0.1:${port}/`);
const form = new FormData();
@@ -284,7 +285,13 @@ async function main() {
if (!ares.ok) throw new Error(`/emissions returned ${ares.status}: ${await ares.text()}`);
emissions = await ares.json();
} else {
- alignerModel = null;
+ // Say which file was looked for: a custom OPENSCREEN_WHISPER_MODEL moves
+ // the cache this path is derived from, and would read as "no aligner".
+ alignerSkipped = !WITH_VAD
+ ? "no VAD model, so no speech to align"
+ : alignerModel
+ ? `no aligner at ${alignerModel} (set OPENSCREEN_ALIGNER_MODEL)`
+ : `no aligner for "${json.detected_language}"`;
}
} finally {
child.kill();
@@ -458,7 +465,7 @@ async function main() {
console.log(`aligner: ${emissions.elapsed_s.toFixed(2)}s on ${emissions.device}`);
}
} else {
- console.log("\n(no aligner for this language in the cache: aligner checks skipped)");
+ console.log(`\n(aligner checks skipped: ${alignerSkipped})`);
}
if (json.timing) {
diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md
index 35b9187af..2467004e6 100644
--- a/technical-documentation/architecture/transcription-and-captions.md
+++ b/technical-documentation/architecture/transcription-and-captions.md
@@ -112,10 +112,9 @@ renderer imposes a timeout that a slow download could trip: the preload does a
bare `ipcRenderer.invoke`, `fetchWithRetry` has no per-request deadline, and
whisper-server's 30 s readiness budget only starts once the download resolved.
-The word aligner (§ Word-level alignment, step 5) is the exception to "first
-run only": it is fetched the first time a transcription detects English or
-French (109 or 348 MB), in the middle of that run, which then simply takes
-longer once more.
+The word aligner (§ Word-level alignment, step 5) is not part of that phase: it
+is fetched in the background the first time a transcription detects English or
+French (109 or 348 MB), and no chunk ever waits for it.
Three edges make that promise hold, and each is load-bearing:
@@ -288,9 +287,9 @@ it verbatim in the response.
midpoint. Next to a pause (a gap of 100 ms or more between letters) the
edges move further out, 60 ms before and 150 ms after, both capped at
mid-pause: early there is silence, late is an audible attack.
- - **Fallback.** No aligner for the language, a download that failed, a
- helper without `/emissions`, a stretch whose letters do not fit in its
- frames: the words keep the times of step 4. Step 6 runs either way.
+ - **Fallback.** No aligner for the language, a download not landed yet or
+ failed, a helper without `/emissions`, a stretch whose letters do not fit in
+ its frames: the words keep the times of step 4. Step 6 runs either way.
6. **Anchor phrase edges on the speech** —
[`electron/stt/snapWordBoundaries.ts`](../../electron/stt/snapWordBoundaries.ts)
leaves boundaries inside a phrase alone: they are within ~16 ms of the audio
@@ -390,10 +389,13 @@ digest an invariant. Bumping the model therefore means bumping the revision
and the digest together.
**The word aligners** (step 5) are fetched only for a language that has one,
-the first time a transcription detects it, inside the same run (`phase:
-"model"`, `model: "ctc-aligner"`), into `stt-models/ctc-aligner/`. A failed
-download keeps whisper's times for the rest of the run and is retried on the
-next one.
+into `stt-models/ctc-aligner/`, and never inside a chunk. The first time a
+session meets a language, a copy already on disk is verified in place (a local
+read). Without one, the download starts in the background
+(`SttManager.alignerFor`): the chunks that run before it lands keep whisper's
+times, the later ones and later transcriptions use it. A download stalled for
+30 s, cancelled with the transcription, or interrupted by quitting is
+abandoned, and retried on the next transcription rather than per chunk.
| Language | Model | Download |
|---|---|---|
@@ -463,7 +465,11 @@ timings present, the DTW guardrail passed, `backend` reporting GPU offload on a
GPU-capable host, `detected_language` resolved rather than echoed, word times
monotonic and inside the clip, and WER against a reference. On macOS it
synthesizes its own clip with `say`, so it needs no fixture; elsewhere pass
-`--wav <file>` (and optionally `--ref "<expected text>"`). This is the check
+`--wav <file>` (and optionally `--ref "<expected text>"`). When the detected
+language's aligner is cached (or `OPENSCREEN_ALIGNER_MODEL` names one), it also
+calls `/emissions` and checks that the answer is well-formed log-probabilities,
+computed on the GPU on a GPU-capable host, decoding to roughly what whisper
+heard (letter error rate), with the aligned words still ordered. This is the check
that the unit tests structurally cannot make: they mock `fetch`, so they assert
against a hand-written fixture rather than the binary.
diff --git a/tools/stt-eval/word-timing/evaluate.mjs b/tools/stt-eval/word-timing/evaluate.mjs
index 4c7cbe873..8a08b718f 100644
--- a/tools/stt-eval/word-timing/evaluate.mjs
+++ b/tools/stt-eval/word-timing/evaluate.mjs
@@ -126,9 +126,12 @@ for (const file of readdirSync(rawDir)
const full = postPass(rawWords, samples, speech);
const tsMs = performance.now() - t0;
const variants = { raw: rawWords, post, "post+vad": full };
- if (json.emissions && speech) {
+ const emissions = json.emissions ? ctc.parseEmissions(json.emissions) : null;
+ if (json.emissions && !emissions)
+ console.warn(`${file}: unreadable emissions, CTC stages skipped`);
+ if (emissions && speech) {
const t1 = performance.now();
- const aligned = ctc.alignWordsOnEmissions(rawWords, speech, ctc.parseEmissions(json.emissions));
+ const aligned = ctc.alignWordsOnEmissions(rawWords, speech, emissions);
variants.ctc = aligned;
variants["ctc+vad"] = postPass(aligned, samples, speech);
ctcMs += performance.now() - t1;
diff --git a/tools/stt-eval/word-timing/real-check.mjs b/tools/stt-eval/word-timing/real-check.mjs
index b0fca0230..a0d9ea90c 100644
--- a/tools/stt-eval/word-timing/real-check.mjs
+++ b/tools/stt-eval/word-timing/real-check.mjs
@@ -66,8 +66,11 @@ const raw = json.segments
.filter((w) => w.word);
const speech = json.speech.map((s) => ({ startSec: s.start, endSec: s.end }));
const post = anchorWordsOnSpeech(raw, speech);
-const aligned = emissions
- ? anchorWordsOnSpeech(alignWordsOnEmissions(raw, speech, parseEmissions(emissions)), speech)
+const parsed = emissions ? parseEmissions(emissions) : null;
+if (emissions && !parsed)
+ throw new Error("/emissions answered something that is not letter scores (vocab/regions/sizes)");
+const aligned = parsed
+ ? anchorWordsOnSpeech(alignWordsOnEmissions(raw, speech, parsed), speech)
: null;
mkdirSync(RESULTS, { recursive: true });
writeFileSync(