Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): read the WAV in JS so Built-in transcription runs inside Electron (#5129)
Miguel Ángelf7bba2288dfc
diff --git a/packages/cli/src/whisper/sherpa.test.ts b/packages/cli/src/whisper/sherpa.test.tsindex 4f935ab457..25f0377020 100644--- a/packages/cli/src/whisper/sherpa.test.ts+++ b/packages/cli/src/whisper/sherpa.test.ts@@ -25,6 +25,7 @@ import {transcribeWithSherpa,type ModelFile,} from "./sherpa.js";+import { encodeWav } from "./wav.test-helpers.js";const file = (name: string, content: string): ModelFile => ({name,@@ -183,6 +184,7 @@ describe("a pinned Sherpa copy beside the CLI", () => {let root: string;beforeEach(() => {root = mkdtempSync(join(tmpdir(), "hf-sherpa-beside-"));+ writeFileSync(join(root, "speech.wav"), encodeWav(new Float32Array(100).fill(0.5), 100));});afterEach(() => {vi.restoreAllMocks();@@ -204,7 +206,7 @@ describe("a pinned Sherpa copy beside the CLI", () => {return `if (process.pid === ${process.pid}) require("node:fs").writeFileSync(${JSON.stringify(parentMarker)}, "");module.exports = {- readWave() { return { sampleRate: 100, samples: new Float32Array(100).fill(0.5) }; },+ readWave() { throw new Error("External buffers are not allowed"); },OfflineRecognizer: class {createStream() { return { acceptWaveform() {} }; }decode() {}@@ -228,7 +230,8 @@ module.exports = {const event = { type: "progress", phase: "transcription", model: "parakeet-tdt-0.6b-v3" };// The worker's one 1 s window streams its words before the final transcript.expect(onEvent.mock.calls).toEqual([- [{ ...event, status: "started", durationSeconds: null }],+ // The started line sizes the WAV as 16 kHz mono: 44 header bytes and 100 samples here.+ [{ ...event, status: "started", durationSeconds: (44 + 2 * 100) / 32_000 }],[{ type: "words", model: event.model, words: result, through: 1 }],[{ ...event, status: "completed" }],]);diff --git a/packages/cli/src/whisper/sherpaWorker.test.ts b/packages/cli/src/whisper/sherpaWorker.test.tsindex a8e6eebce7..7f022137cb 100644--- a/packages/cli/src/whisper/sherpaWorker.test.ts+++ b/packages/cli/src/whisper/sherpaWorker.test.ts@@ -5,23 +5,20 @@ import { join } from "node:path";import { fileURLToPath } from "node:url";import { afterEach, describe, expect, it } from "vitest";import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX, SHERPA_WINDOW_PREFIX } from "./parakeet.js";+import { encodeWav } from "./wav.test-helpers.js";const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url));// Stand-in sherpa-onnx-node, 3 s of sound: the whole window (300 samples) drops the first 2 s; the// padded window (350) hears it, and for speech.wav so does the padded gap +-0.5 s (308) alone.const FAKE_SHERPA = `-let path;+const path = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT).wavPath;module.exports = {- readWave(p) {- path = p;- if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path);- if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n ../a.node\\n ./b.node\\n");- if (path.endsWith("long.wav")) return { sampleRate: 100, samples: new Float32Array(60000) };- return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) };- },+ // What Electron's V8 sandbox throws for this reader's native buffer: the worker reads the WAV itself.+ readWave() { throw new Error("External buffers are not allowed"); },OfflineRecognizer: class {constructor() {+ if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n ../a.node\\n ./b.node\\n");if (process.env.STARTED) require("fs").writeFileSync(process.env.STARTED, "");}createStream() { return { acceptWaveform(w) { this.w = w; } }; }@@ -69,13 +66,20 @@ describe("sherpaWorker", () => {return root;}+ /** 3 s of steady sound at 100 Hz, as the stand-in decoder expects; long.wav is 600 s of silence. */+ function wav(name: string): string {+ const samples = name === "long.wav" ? new Float32Array(60000) : new Float32Array(300).fill(0.5);+ writeFileSync(join(root, name), encodeWav(samples, 100));+ return join(root, name);+ }+function windowsOf(stdout: string) {const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!;return JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length));}it("decodes a gap that skipped loud audio on its own and splices in its new tokens", async () => {- const { code, stdout } = await runWorker(fakeRuntime(), "speech.wav");+ const { code, stdout } = await runWorker(fakeRuntime(), wav("speech.wav"));expect(code).toBe(0);expect(windowsOf(stdout)).toEqual([{ offset: 0, tokens: [" ask", " not"], timestamps: [0.5, 2.08], durations: [0.4, 0.4] },@@ -83,7 +87,7 @@ describe("sherpaWorker", () => {});it("prints each window as it is decoded, with the audio seconds done", async () => {- const { stdout } = await runWorker(fakeRuntime(), "speech.wav");+ const { stdout } = await runWorker(fakeRuntime(), wav("speech.wav"));const streamed = stdout.split("\n").filter((l) => l.startsWith(SHERPA_WINDOW_PREFIX))@@ -92,7 +96,7 @@ describe("sherpaWorker", () => {});it("re-decodes the window with leading silence when the gap alone gives nothing", async () => {- const { code, stdout } = await runWorker(fakeRuntime(), "hum.wav");+ const { code, stdout } = await runWorker(fakeRuntime(), wav("hum.wav"));expect(code).toBe(0);expect(windowsOf(stdout)).toEqual([{ offset: 0, tokens: [" ask", " not"], timestamps: [0.25, 1], durations: [0.4, 0.4] },@@ -100,9 +104,11 @@ describe("sherpaWorker", () => {});it("exits non-zero with one prefixed error line when it cannot read the audio", async () => {- const { code, stderr } = await runWorker(fakeRuntime(), "broken.wav");+ const broken = join(fakeRuntime(), "broken.wav");+ writeFileSync(broken, "not audio");+ const { code, stderr } = await runWorker(root, broken);expect(code).toBe(1);- expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Failed to read broken.wav`);+ expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}${broken} is not a 16-bit mono PCM WAV`);});it("keeps every line of a multi-line error on its one prefixed line", async () => {@@ -116,7 +122,7 @@ describe("sherpaWorker", () => {async () => {const started = join(fakeRuntime(), "started");const input = JSON.stringify({- wavPath: "long.wav",+ wavPath: wav("long.wav"),runtimePath: join(root, "node_modules", "sherpa-onnx-node", "index.js"),config: {},});diff --git a/packages/cli/src/whisper/sherpaWorker.ts b/packages/cli/src/whisper/sherpaWorker.tsindex a5b96368e9..3aec04cd55 100644--- a/packages/cli/src/whisper/sherpaWorker.ts+++ b/packages/cli/src/whisper/sherpaWorker.ts@@ -8,6 +8,7 @@ import {SHERPA_WINDOW_PREFIX,type SherpaWindow,} from "./parakeet.js";+import { readWav } from "./wav.js";interface Wave {samples: Float32Array;@@ -15,7 +16,6 @@ interface Wave {}interface SherpaOnnx {- readWave(path: string): Wave;OfflineRecognizer: new (config: object) => {createStream(): { acceptWaveform(wave: Wave): void };decode(stream: unknown): void;@@ -49,7 +49,7 @@ function decodeWindow(try {const sherpa = createRequire(import.meta.url)(runtimePath) as SherpaOnnx;const recognizer = new sherpa.OfflineRecognizer(config);- const wave = sherpa.readWave(wavPath);+ const wave = readWav(wavPath);const cuts = silenceCuts(wave.samples, wave.sampleRate);const windows: SherpaWindow[] = [];for (let k = 0; k + 1 < cuts.length; k++) {diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.tsindex 5f590d5f6e..caac298f65 100644--- a/packages/cli/src/whisper/transcribe.ts+++ b/packages/cli/src/whisper/transcribe.ts@@ -10,22 +10,10 @@ import { randomUUID } from "node:crypto";import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";import { stoppedByCancelSignal } from "../utils/renderCancellation.js";import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";+import { findWavChunk } from "./wav.js";import type { Word } from "./normalize.js";import { emitWords } from "./progress.js";-function findWavDataChunk(buf: Buffer): { offset: number; size: number } | null {- if (buf.length < 12) return null;- let pos = 12; // skip RIFF header- while (pos + 8 < buf.length) {- const id = buf.toString("ascii", pos, pos + 4);- const size = buf.readUInt32LE(pos + 4);- if (id === "data") return { offset: pos + 8, size: Math.min(size, buf.length - pos - 8) };- pos += 8 + size;- if (size % 2 !== 0) pos++; // RIFF chunks are word-aligned- }- return null;-}-const WHISPER_TIMEOUT_FLOOR_MS = 300_000;const WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS = 10_000;const WHISPER_TIMEOUT_CAP_MS = 43_200_000;@@ -196,7 +184,7 @@ export function detectSpeechOnset(wavPath: string): number | null {try {const buf = readFileSync(wavPath);- const dataChunk = findWavDataChunk(buf);+ const dataChunk = findWavChunk(buf, "data");if (!dataChunk) return null;const pcm = new Int16Array(buf.buffer, buf.byteOffset + dataChunk.offset, dataChunk.size / 2);const totalWindows = Math.floor(pcm.length / WINDOW_SAMPLES);diff --git a/packages/cli/src/whisper/wav.test-helpers.ts b/packages/cli/src/whisper/wav.test-helpers.tsnew file mode 100644index 0000000000..d0f924b63d--- /dev/null+++ b/packages/cli/src/whisper/wav.test-helpers.ts@@ -0,0 +1,21 @@+/** A 16-bit mono PCM WAV of `samples` (-1..1), as prepareWav writes one. */+export function encodeWav(samples: ArrayLike<number>, sampleRate: number): Buffer {+ const data = Buffer.alloc(samples.length * 2);+ for (let i = 0; i < samples.length; i++) {+ data.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(samples[i]! * 32768))), 2 * i);+ }+ const head = Buffer.alloc(44);+ head.write("RIFF", 0, "ascii");+ head.writeUInt32LE(36 + data.length, 4);+ head.write("WAVEfmt ", 8, "ascii");+ head.writeUInt32LE(16, 16);+ head.writeUInt16LE(1, 20);+ head.writeUInt16LE(1, 22);+ head.writeUInt32LE(sampleRate, 24);+ head.writeUInt32LE(sampleRate * 2, 28);+ head.writeUInt16LE(2, 32);+ head.writeUInt16LE(16, 34);+ head.write("data", 36, "ascii");+ head.writeUInt32LE(data.length, 40);+ return Buffer.concat([head, data]);+}diff --git a/packages/cli/src/whisper/wav.test.ts b/packages/cli/src/whisper/wav.test.tsnew file mode 100644index 0000000000..320abb4bd5--- /dev/null+++ b/packages/cli/src/whisper/wav.test.ts@@ -0,0 +1,35 @@+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterEach, beforeEach, expect, it } from "vitest";+import { readWav } from "./wav.js";+import { encodeWav } from "./wav.test-helpers.js";++let dir: string;+beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), "hf-wav-"));+});+afterEach(() => rmSync(dir, { recursive: true, force: true }));+const file = (name: string, bytes: Buffer) => {+ writeFileSync(join(dir, name), bytes);+ return join(dir, name);+};++it("reads a 16-bit mono WAV's samples and rate in JS", () => {+ const wav = readWav(file("a.wav", encodeWav([0, 0.5, -0.5, -1], 16_000)));+ expect(wav.sampleRate).toBe(16_000);+ expect([...wav.samples]).toEqual([0, 0.5, -0.5, -1]);+});++it("reads a WAV with no samples as silence, as ffmpeg writes for empty audio", () => {+ expect(readWav(file("empty.wav", encodeWav([], 16_000))).samples).toHaveLength(0);+});++it("names a file that is not a 16-bit mono WAV", () => {+ const stereo = encodeWav([0, 0], 16_000);+ stereo.writeUInt16LE(2, 22);+ expect(() => readWav(file("text.wav", Buffer.from("not audio")))).toThrow(+ "text.wav is not a 16-bit mono PCM WAV",+ );+ expect(() => readWav(file("stereo.wav", stereo))).toThrow("is not a 16-bit mono PCM WAV");+});diff --git a/packages/cli/src/whisper/wav.ts b/packages/cli/src/whisper/wav.tsnew file mode 100644index 0000000000..e2d84e2d09--- /dev/null+++ b/packages/cli/src/whisper/wav.ts@@ -0,0 +1,40 @@+import { readFileSync } from "node:fs";++/** Where a RIFF chunk's bytes start and how many there are, or null when the file has none. */+export function findWavChunk(buf: Buffer, want: string): { offset: number; size: number } | null {+ if (buf.length < 12) return null;+ let pos = 12; // skip RIFF header+ while (pos + 8 <= buf.length) {+ const id = buf.toString("ascii", pos, pos + 4);+ const size = buf.readUInt32LE(pos + 4);+ if (id === want) return { offset: pos + 8, size: Math.min(size, buf.length - pos - 8) };+ pos += 8 + size;+ if (size % 2 !== 0) pos++; // RIFF chunks are word-aligned+ }+ return null;+}++const PCM = 1;+const EXTENSIBLE = 0xfffe;++/** A prepared 16-bit mono WAV, read in JS: sherpa's readWave returns memory Electron refuses. */+export function readWav(path: string): { samples: Float32Array; sampleRate: number } {+ const buf = readFileSync(path);+ const riff = buf.toString("ascii", 0, 4) === "RIFF" && buf.toString("ascii", 8, 12) === "WAVE";+ const fmt = riff ? findWavChunk(buf, "fmt ") : null;+ const data = fmt && findWavChunk(buf, "data");+ const format = fmt && buf.readUInt16LE(fmt.offset);+ if (+ !fmt ||+ !data ||+ (format !== PCM && format !== EXTENSIBLE) ||+ buf.readUInt16LE(fmt.offset + 2) !== 1 ||+ buf.readUInt16LE(fmt.offset + 14) !== 16+ ) {+ throw new Error(`${path} is not a 16-bit mono PCM WAV`);+ }+ const samples = new Float32Array(data.size >> 1);+ for (let i = 0; i < samples.length; i++)+ samples[i] = buf.readInt16LE(data.offset + 2 * i) / 32768;+ return { samples, sampleRate: buf.readUInt32LE(fmt.offset + 4) };+}