Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): preserve transcribed words before speech onset (#5164)
Miguel Ángele21ab020da17
diff --git a/packages/cli/src/commands/transcribe.test.ts b/packages/cli/src/commands/transcribe.test.tsindex 4c444dbdf2..7657d034bb 100644--- a/packages/cli/src/commands/transcribe.test.ts+++ b/packages/cli/src/commands/transcribe.test.ts@@ -112,6 +112,34 @@ describe("transcribe command", () => {vi.unstubAllEnvs();});+ it("keeps quiet Spanish words before the relative speech onset in transcript and captions", async () => {+ const { dir, input } = dummyAudio();+ dirs.push(dir);+ const words = [+ { id: "w0", text: "Siempre", start: 0.2, end: 0.49 },+ { id: "w1", text: "que", start: 0.49, end: 0.7 },+ { id: "w2", text: "los", start: 0.7, end: 0.91 },+ { id: "w3", text: "nietos", start: 0.91, end: 1.33 },+ { id: "w4", text: "vecindario", start: 10.03, end: 11.02 },+ ];+ const htmlPath = join(dir, "index.html");+ writeFileSync(htmlPath, "<script>const TRANSCRIPT = [];</script>");+ transcribeMock.mockImplementation(async (_input, outputDir) => {+ const result = fakeTranscript(outputDir, "whisper");+ writeFileSync(result.transcriptPath, JSON.stringify(words));+ return { ...result, wordCount: 5, durationSeconds: 50, speechOnsetSeconds: 10 };+ });++ await runCommand(transcribeCmd, {+ rawArgs: [input, "--dir", dir, "--engine", "whisper", "--language", "es", "--json"],+ });++ expect(JSON.parse(readFileSync(join(dir, "transcript.json"), "utf8"))).toEqual(words);+ const captions = readFileSync(htmlPath, "utf8").match(/const TRANSCRIPT = ([\s\S]*?);/)![1]!;+ expect(JSON.parse(captions)).toEqual(words);+ expect(lastJson()).toMatchObject({ ok: true, wordCount: 5, speechOnsetSeconds: 10 });+ });+it("keeps typed progress on stderr and reports resolved metadata in one stdout result", async () => {const { dir, input } = dummyAudio();dirs.push(dir);diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.tsindex c37e14b7b9..56f4856ab4 100644--- a/packages/cli/src/commands/transcribe.ts+++ b/packages/cli/src/commands/transcribe.ts@@ -309,8 +309,7 @@ async function transcribeAudio(},): Promise<void> {const { transcribe } = await import("../whisper/transcribe.js");- const { loadTranscript, patchCaptionHtml, stripBeforeOnset } =- await import("../whisper/normalize.js");+ const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");const {DecodeCancelled,@@ -411,18 +410,7 @@ async function transcribeAudio(}}- let { words } = loadTranscript(result.transcriptPath);-- if (result.speechOnsetSeconds != null) {- const before = words.length;- words = stripBeforeOnset(words, result.speechOnsetSeconds);- const stripped = before - words.length;- if (stripped > 0 && !opts.json) {- spin?.message(- `Stripped ${stripped} words before speech onset at ${result.speechOnsetSeconds.toFixed(1)}s`,- );- }- }+ const { words } = loadTranscript(result.transcriptPath);writeFileSync(result.transcriptPath, JSON.stringify(words, null, 2));patchCaptionHtml(dir, words);diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.tsindex 4439d00a6e..969bbff3b0 100644--- a/packages/cli/src/whisper/normalize.test.ts+++ b/packages/cli/src/whisper/normalize.test.ts@@ -6,7 +6,6 @@ import {loadTranscript,detectFormat,patchCaptionHtml,- stripBeforeOnset,formatSrt,formatVtt,wordsToCues,@@ -671,44 +670,3 @@ describe("detectSpeechOnset", () => {expect(onset).toBeNull();});});--describe("stripBeforeOnset", () => {- it("removes words before onset time", () => {- const words = [- { text: "ghost", start: 0.5, end: 2.0 },- { text: "alone", start: 3.0, end: 5.0 },- { text: "Given", start: 19.0, end: 19.5 },- { text: "the", start: 19.5, end: 20.0 },- ];- const result = stripBeforeOnset(words, 18.5);- expect(result).toHaveLength(2);- expect(result[0]!.text).toBe("Given");- });-- it("keeps words within 0.5s tolerance of onset", () => {- const words = [- { text: "hello", start: 18.2, end: 18.8 },- { text: "world", start: 19.0, end: 19.5 },- ];- const result = stripBeforeOnset(words, 18.5);- expect(result).toHaveLength(2);- });-- it("keeps everything when onset is 0", () => {- const words = [- { text: "hello", start: 0.1, end: 0.5 },- { text: "world", start: 0.6, end: 1.0 },- ];- const result = stripBeforeOnset(words, 0);- expect(result).toHaveLength(2);- });-- it("returns empty array when all words are before onset", () => {- const words = [- { text: "ghost", start: 0.5, end: 2.0 },- { text: "alone", start: 3.0, end: 5.0 },- ];- const result = stripBeforeOnset(words, 20.0);- expect(result).toHaveLength(0);- });-});diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.tsindex 5c7bb29a2c..6683d634af 100644--- a/packages/cli/src/whisper/normalize.ts+++ b/packages/cli/src/whisper/normalize.ts@@ -525,16 +525,6 @@ export function loadTranscript(filePath: string): { words: Word[]; format: Transreturn { words, format };}-/**- * Remove words that fall before the detected speech onset.- * Whisper can hallucinate words over non-speech sections at the start of audio.- */-export function stripBeforeOnset(words: Word[], onsetSeconds: number): Word[] {- // 0.5s tolerance: keep words whose timestamps straddle the onset boundary,- // since whisper may assign a slightly early start to the first spoken word.- return words.filter((w) => w.start >= onsetSeconds - 0.5);-}-export function patchCaptionHtml(dir: string, words: Word[]): void {if (words.length === 0) return;