Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): preserve voice placement when ducking empty transcripts (#5271)
Abdullah M. Nazif320de3973bd1
diff --git a/packages/cli/src/media-use/lib/duck.mjs b/packages/cli/src/media-use/lib/duck.mjsindex 9bbe7afda0..dcf81774f3 100644--- a/packages/cli/src/media-use/lib/duck.mjs+++ b/packages/cli/src/media-use/lib/duck.mjs@@ -5,7 +5,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,const merge = Number(mergeGap);const lists = wordListsFromMediaMeta(meta);const voices = Array.isArray(meta?.voices) ? meta.voices : [];- if (lists.length > 1 && !offsets && !sequential) {+ if (lists.filter((words) => words.length > 0).length > 1 && !offsets && !sequential) {throw new Error("audio_meta has multiple voice lines with file-relative times; pass --sequential or --offsets so spans land at composition time",);@@ -16,6 +16,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,const voice = voices[i];let offset = 0;if (offsets) {+ if (lists[i].length === 0) continue;const id = voice?.id ?? String(i);if (!(id in offsets)) throw new Error(`--offsets is missing voice "${id}"`);offset = Number(offsets[id]) || 0;diff --git a/packages/cli/src/media-use/lib/duck.test.mjs b/packages/cli/src/media-use/lib/duck.test.mjsindex 471eafff6d..164c12bbdc 100644--- a/packages/cli/src/media-use/lib/duck.test.mjs+++ b/packages/cli/src/media-use/lib/duck.test.mjs@@ -75,6 +75,63 @@ test("speechSpans returns empty spans for empty input", () => {assert.deepEqual(speechSpans({ voices: [] }, { mergeGap: 0.6 }), []);});+test("speechSpans offsets retain voice IDs across empty transcripts", () => {+ const meta = {+ voices: [+ { id: "silent", duration_s: 2, words: [] },+ { id: "narration", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },+ ],+ };+ const expected = [{ start: 4.1, end: 4.9 }];+ assert.deepEqual(speechSpans(meta, { offsets: { silent: 0, narration: 4 } }), expected);+ assert.deepEqual(speechSpans(meta, { offsets: { narration: 4 } }), expected);+ assert.throws(() => speechSpans(meta, { offsets: { silent: 0 } }), /missing voice "narration"/);+});++test("speechSpans sequential counts empty line durations and gaps", () => {+ const meta = {+ voices: [+ { id: "intro", duration_s: 2, words: [] },+ { id: "first", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },+ { id: "pause", duration_s: 3, words: [] },+ { id: "last", duration_s: 1, words: [word("w1", "goodbye", 0.1, 0.9)] },+ ],+ };+ assert.deepEqual(speechSpans(meta, { sequential: true, gap: 0.5 }), [+ { start: 2.6, end: 3.4 },+ { start: 7.6, end: 8.4 },+ ]);+});++test("speechSpans offsets without IDs retain original voice indices", () => {+ const meta = {+ voices: [{ duration_s: 2 }, { duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] }],+ };+ assert.deepEqual(speechSpans(meta, { offsets: { 1: 4 } }), [{ start: 4.1, end: 4.9 }]);+});++test("speechSpans keeps normalized-empty transcripts aligned with their voices", () => {+ const meta = {+ voices: [+ { id: "untimed", duration_s: 2, words: [{ text: "hello" }] },+ { id: "narration", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },+ ],+ };+ assert.deepEqual(speechSpans(meta, { sequential: true }), [{ start: 2.1, end: 2.9 }]);+});++test("speechSpans does not require placement for transcripts without speech", () => {+ const meta = {+ voices: [+ { id: "a", duration_s: 2, words: [] },+ { id: "b", duration_s: 1 },+ ],+ };+ assert.deepEqual(speechSpans(meta), []);+ assert.deepEqual(speechSpans(meta, { offsets: {} }), []);+ assert.deepEqual(speechSpans(meta, { sequential: true }), []);+});+test("duckKeyframes shapes attack and release from base volume", () => {assert.deepEqual(duckKeyframes([{ start: 3, end: 5 }], {@@ -148,6 +205,36 @@ test("--json spans match --merge-gap semantics exactly", () => {}});+for (const [placement, flags, expected] of [+ ["sequential", ["--sequential", "--gap", "0.5"], [{ start: 2.6, end: 3.4 }]],+ ["offsets", ["--offsets", "narration=4"], [{ start: 4.1, end: 4.9 }]],+]) {+ test(`audio-duck --json preserves empty voice placement with ${placement}`, (t) => {+ const dir = mkdtempSync(join(tmpdir(), "media-use-duck-empty-"));+ t.after(() => rmSync(dir, { recursive: true, force: true }));+ const metaPath = join(dir, "audio_meta.json");+ writeFileSync(+ metaPath,+ JSON.stringify({+ voices: [+ { id: "silent", duration_s: 2, words: [] },+ { id: "narration", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },+ ],+ }),+ );+ const result = JSON.parse(+ execFileSync(+ process.execPath,+ [SCRIPT, "--meta", metaPath, "--target", "#bgm", ...flags, "--json"],+ { encoding: "utf8" },+ ),+ );+ assert.deepEqual(result.spans, expected);+ assert.equal(result.keyframes[0].time, expected[0].start);+ assert.equal(result.lane.lanes[0].points[1].t, expected[0].start);+ });+}+function word(id, text, start, end) {return { id, text, start, end };}diff --git a/packages/cli/src/media-use/lib/words.mjs b/packages/cli/src/media-use/lib/words.mjsindex 114cf766ce..e062ecbfef 100644--- a/packages/cli/src/media-use/lib/words.mjs+++ b/packages/cli/src/media-use/lib/words.mjs@@ -14,5 +14,5 @@ export function normalizeWords(input) {export function wordListsFromMediaMeta(input) {if (Array.isArray(input) || Array.isArray(input?.words)) return [normalizeWords(input)];if (!Array.isArray(input?.voices)) return [];- return input.voices.map((voice) => normalizeWords(voice)).filter((words) => words.length > 0);+ return input.voices.map((voice) => normalizeWords(voice));}diff --git a/skills-manifest.json b/skills-manifest.jsonindex 8d35dfe882..8c12b61295 100644--- a/skills-manifest.json+++ b/skills-manifest.json@@ -54,7 +54,7 @@"files": 1},"media-use": {- "hash": "2bfc140b0b645f66",+ "hash": "c8800249da2d1ce6","files": 109},"motion-graphics": {diff --git a/skills/media-use/scripts/lib/duck.mjs b/skills/media-use/scripts/lib/duck.mjsindex 9bbe7afda0..dcf81774f3 100644--- a/skills/media-use/scripts/lib/duck.mjs+++ b/skills/media-use/scripts/lib/duck.mjs@@ -5,7 +5,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,const merge = Number(mergeGap);const lists = wordListsFromMediaMeta(meta);const voices = Array.isArray(meta?.voices) ? meta.voices : [];- if (lists.length > 1 && !offsets && !sequential) {+ if (lists.filter((words) => words.length > 0).length > 1 && !offsets && !sequential) {throw new Error("audio_meta has multiple voice lines with file-relative times; pass --sequential or --offsets so spans land at composition time",);@@ -16,6 +16,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,const voice = voices[i];let offset = 0;if (offsets) {+ if (lists[i].length === 0) continue;const id = voice?.id ?? String(i);if (!(id in offsets)) throw new Error(`--offsets is missing voice "${id}"`);offset = Number(offsets[id]) || 0;diff --git a/skills/media-use/scripts/lib/words.mjs b/skills/media-use/scripts/lib/words.mjsindex 114cf766ce..e062ecbfef 100644--- a/skills/media-use/scripts/lib/words.mjs+++ b/skills/media-use/scripts/lib/words.mjs@@ -14,5 +14,5 @@ export function normalizeWords(input) {export function wordListsFromMediaMeta(input) {if (Array.isArray(input) || Array.isArray(input?.words)) return [normalizeWords(input)];if (!Array.isArray(input?.voices)) return [];- return input.voices.map((voice) => normalizeWords(voice)).filter((words) => words.length > 0);+ return input.voices.map((voice) => normalizeWords(voice));}