Skip to content

Write HTML. Render video. Built for agents.

fix(cli): preserve voice placement when ducking empty transcripts (#5271)

Abdullah M. Nazif320de3973bd1
diff --git a/packages/cli/src/media-use/lib/duck.mjs b/packages/cli/src/media-use/lib/duck.mjs
index 9bbe7afda0..dcf81774f3 100644
--- a/packages/cli/src/media-use/lib/duck.mjs
+++ b/packages/cli/src/media-use/lib/duck.mjs
@@ -5,7 +5,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,
const merge = Number(mergeGap);
const lists = wordListsFromMediaMeta(meta);
const voices = Array.isArray(meta?.voices) ? meta.voices : [];
- if (lists.length > 1 && !offsets && !sequential) {
+ if (lists.filter((words) => words.length > 0).length > 1 && !offsets && !sequential) {
throw new Error(
"audio_meta has multiple voice lines with file-relative times; pass --sequential or --offsets so spans land at composition time",
);
@@ -16,6 +16,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,
const voice = voices[i];
let offset = 0;
if (offsets) {
+ if (lists[i].length === 0) continue;
const id = voice?.id ?? String(i);
if (!(id in offsets)) throw new Error(`--offsets is missing voice "${id}"`);
offset = Number(offsets[id]) || 0;
diff --git a/packages/cli/src/media-use/lib/duck.test.mjs b/packages/cli/src/media-use/lib/duck.test.mjs
index 471eafff6d..164c12bbdc 100644
--- a/packages/cli/src/media-use/lib/duck.test.mjs
+++ b/packages/cli/src/media-use/lib/duck.test.mjs
@@ -75,6 +75,63 @@ test("speechSpans returns empty spans for empty input", () => {
assert.deepEqual(speechSpans({ voices: [] }, { mergeGap: 0.6 }), []);
});
+test("speechSpans offsets retain voice IDs across empty transcripts", () => {
+ const meta = {
+ voices: [
+ { id: "silent", duration_s: 2, words: [] },
+ { id: "narration", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },
+ ],
+ };
+ const expected = [{ start: 4.1, end: 4.9 }];
+ assert.deepEqual(speechSpans(meta, { offsets: { silent: 0, narration: 4 } }), expected);
+ assert.deepEqual(speechSpans(meta, { offsets: { narration: 4 } }), expected);
+ assert.throws(() => speechSpans(meta, { offsets: { silent: 0 } }), /missing voice "narration"/);
+});
+
+test("speechSpans sequential counts empty line durations and gaps", () => {
+ const meta = {
+ voices: [
+ { id: "intro", duration_s: 2, words: [] },
+ { id: "first", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },
+ { id: "pause", duration_s: 3, words: [] },
+ { id: "last", duration_s: 1, words: [word("w1", "goodbye", 0.1, 0.9)] },
+ ],
+ };
+ assert.deepEqual(speechSpans(meta, { sequential: true, gap: 0.5 }), [
+ { start: 2.6, end: 3.4 },
+ { start: 7.6, end: 8.4 },
+ ]);
+});
+
+test("speechSpans offsets without IDs retain original voice indices", () => {
+ const meta = {
+ voices: [{ duration_s: 2 }, { duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] }],
+ };
+ assert.deepEqual(speechSpans(meta, { offsets: { 1: 4 } }), [{ start: 4.1, end: 4.9 }]);
+});
+
+test("speechSpans keeps normalized-empty transcripts aligned with their voices", () => {
+ const meta = {
+ voices: [
+ { id: "untimed", duration_s: 2, words: [{ text: "hello" }] },
+ { id: "narration", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },
+ ],
+ };
+ assert.deepEqual(speechSpans(meta, { sequential: true }), [{ start: 2.1, end: 2.9 }]);
+});
+
+test("speechSpans does not require placement for transcripts without speech", () => {
+ const meta = {
+ voices: [
+ { id: "a", duration_s: 2, words: [] },
+ { id: "b", duration_s: 1 },
+ ],
+ };
+ assert.deepEqual(speechSpans(meta), []);
+ assert.deepEqual(speechSpans(meta, { offsets: {} }), []);
+ assert.deepEqual(speechSpans(meta, { sequential: true }), []);
+});
+
test("duckKeyframes shapes attack and release from base volume", () => {
assert.deepEqual(
duckKeyframes([{ start: 3, end: 5 }], {
@@ -148,6 +205,36 @@ test("--json spans match --merge-gap semantics exactly", () => {
}
});
+for (const [placement, flags, expected] of [
+ ["sequential", ["--sequential", "--gap", "0.5"], [{ start: 2.6, end: 3.4 }]],
+ ["offsets", ["--offsets", "narration=4"], [{ start: 4.1, end: 4.9 }]],
+]) {
+ test(`audio-duck --json preserves empty voice placement with ${placement}`, (t) => {
+ const dir = mkdtempSync(join(tmpdir(), "media-use-duck-empty-"));
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
+ const metaPath = join(dir, "audio_meta.json");
+ writeFileSync(
+ metaPath,
+ JSON.stringify({
+ voices: [
+ { id: "silent", duration_s: 2, words: [] },
+ { id: "narration", duration_s: 1, words: [word("w0", "hello", 0.1, 0.9)] },
+ ],
+ }),
+ );
+ const result = JSON.parse(
+ execFileSync(
+ process.execPath,
+ [SCRIPT, "--meta", metaPath, "--target", "#bgm", ...flags, "--json"],
+ { encoding: "utf8" },
+ ),
+ );
+ assert.deepEqual(result.spans, expected);
+ assert.equal(result.keyframes[0].time, expected[0].start);
+ assert.equal(result.lane.lanes[0].points[1].t, expected[0].start);
+ });
+}
+
function word(id, text, start, end) {
return { id, text, start, end };
}
diff --git a/packages/cli/src/media-use/lib/words.mjs b/packages/cli/src/media-use/lib/words.mjs
index 114cf766ce..e062ecbfef 100644
--- a/packages/cli/src/media-use/lib/words.mjs
+++ b/packages/cli/src/media-use/lib/words.mjs
@@ -14,5 +14,5 @@ export function normalizeWords(input) {
export function wordListsFromMediaMeta(input) {
if (Array.isArray(input) || Array.isArray(input?.words)) return [normalizeWords(input)];
if (!Array.isArray(input?.voices)) return [];
- return input.voices.map((voice) => normalizeWords(voice)).filter((words) => words.length > 0);
+ return input.voices.map((voice) => normalizeWords(voice));
}
diff --git a/skills-manifest.json b/skills-manifest.json
index 8d35dfe882..8c12b61295 100644
--- a/skills-manifest.json
+++ b/skills-manifest.json
@@ -54,7 +54,7 @@
"files": 1
},
"media-use": {
- "hash": "2bfc140b0b645f66",
+ "hash": "c8800249da2d1ce6",
"files": 109
},
"motion-graphics": {
diff --git a/skills/media-use/scripts/lib/duck.mjs b/skills/media-use/scripts/lib/duck.mjs
index 9bbe7afda0..dcf81774f3 100644
--- a/skills/media-use/scripts/lib/duck.mjs
+++ b/skills/media-use/scripts/lib/duck.mjs
@@ -5,7 +5,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,
const merge = Number(mergeGap);
const lists = wordListsFromMediaMeta(meta);
const voices = Array.isArray(meta?.voices) ? meta.voices : [];
- if (lists.length > 1 && !offsets && !sequential) {
+ if (lists.filter((words) => words.length > 0).length > 1 && !offsets && !sequential) {
throw new Error(
"audio_meta has multiple voice lines with file-relative times; pass --sequential or --offsets so spans land at composition time",
);
@@ -16,6 +16,7 @@ export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false,
const voice = voices[i];
let offset = 0;
if (offsets) {
+ if (lists[i].length === 0) continue;
const id = voice?.id ?? String(i);
if (!(id in offsets)) throw new Error(`--offsets is missing voice "${id}"`);
offset = Number(offsets[id]) || 0;
diff --git a/skills/media-use/scripts/lib/words.mjs b/skills/media-use/scripts/lib/words.mjs
index 114cf766ce..e062ecbfef 100644
--- a/skills/media-use/scripts/lib/words.mjs
+++ b/skills/media-use/scripts/lib/words.mjs
@@ -14,5 +14,5 @@ export function normalizeWords(input) {
export function wordListsFromMediaMeta(input) {
if (Array.isArray(input) || Array.isArray(input?.words)) return [normalizeWords(input)];
if (!Array.isArray(input?.voices)) return [];
- return input.voices.map((voice) => normalizeWords(voice)).filter((words) => words.length > 0);
+ return input.voices.map((voice) => normalizeWords(voice));
}