Skip to content

Write HTML. Render video. Built for agents.

fix(media-use): keep frame precision on transcript-cut seek boundaries (#5426)

yi111b71c9f4e1b32
diff --git a/packages/cli/src/media-use/lib/cutlist.mjs b/packages/cli/src/media-use/lib/cutlist.mjs
index 07d0d34159..9cc7d139b6 100644
--- a/packages/cli/src/media-use/lib/cutlist.mjs
+++ b/packages/cli/src/media-use/lib/cutlist.mjs
@@ -148,7 +148,7 @@ function clampRanges(ranges, duration) {
function mergeRanges(ranges) {
const sorted = ranges
- .map((range) => ({ start: round3(range.start), end: round3(range.end) }))
+ .map((range) => ({ start: round6(range.start), end: round6(range.end) }))
.sort((a, b) => a.start - b.start || a.end - b.end);
const merged = [];
for (const range of sorted) {
@@ -173,12 +173,19 @@ function invertRanges(removals, duration) {
return kept;
}
+// Range bounds are rounded only to shed float noise, never to the millisecond:
+// these values become ffmpeg's -ss/-to, where a millisecond is wide enough to
+// step a boundary onto the next frame at 29.97/30/60 fps.
function finalizeKept(ranges) {
return mergeRanges(ranges)
- .map((range) => ({ start: round3(range.start), end: round3(range.end) }))
+ .map((range) => ({ start: round6(range.start), end: round6(range.end) }))
.filter((range) => round3(range.end - range.start) >= MIN_SEGMENT_SECONDS);
}
+function round6(n) {
+ return Math.round(Number(n) * 1e6) / 1e6;
+}
+
function round3(n) {
return Math.round(Number(n) * 1000) / 1000;
}
diff --git a/packages/cli/src/media-use/lib/cutlist.test.mjs b/packages/cli/src/media-use/lib/cutlist.test.mjs
index 98e7710a27..3ae34a9129 100644
--- a/packages/cli/src/media-use/lib/cutlist.test.mjs
+++ b/packages/cli/src/media-use/lib/cutlist.test.mjs
@@ -113,6 +113,18 @@ test("--keep is inverse mode and coalesces direct kept ranges", () => {
]);
});
+test("kept ranges keep frame precision instead of rounding to the millisecond", () => {
+ const transcript = [word("w0", "start", 0, 1), word("w1", "end", 14, 15)];
+
+ // Frame 386 at 30 fps is 12.866666 s. These bounds become ffmpeg's -ss/-to, so
+ // rounding them to 12.867 hands ffmpeg a time just past the frame and the
+ // segment starts one frame late.
+ assert.deepEqual(compileCutList(transcript, { keep: "0-1,12.866666666666667-14" }), [
+ { start: 0, end: 1 },
+ { start: 12.866667, end: 14 },
+ ]);
+});
+
test("--plan on a fixture transcript prints the exact segment JSON", () => {
const dir = mkdtempSync(join(tmpdir(), "media-use-cutlist-"));
try {
diff --git a/skills-manifest.json b/skills-manifest.json
index afc52f5b0d..c62c5e4af4 100644
--- a/skills-manifest.json
+++ b/skills-manifest.json
@@ -54,7 +54,7 @@
"files": 1
},
"media-use": {
- "hash": "0b6e01d1fecb39d6",
+ "hash": "4b6ec7ec0ddaf32e",
"files": 112
},
"motion-graphics": {
diff --git a/skills/media-use/scripts/lib/cutlist.mjs b/skills/media-use/scripts/lib/cutlist.mjs
index 07d0d34159..9cc7d139b6 100644
--- a/skills/media-use/scripts/lib/cutlist.mjs
+++ b/skills/media-use/scripts/lib/cutlist.mjs
@@ -148,7 +148,7 @@ function clampRanges(ranges, duration) {
function mergeRanges(ranges) {
const sorted = ranges
- .map((range) => ({ start: round3(range.start), end: round3(range.end) }))
+ .map((range) => ({ start: round6(range.start), end: round6(range.end) }))
.sort((a, b) => a.start - b.start || a.end - b.end);
const merged = [];
for (const range of sorted) {
@@ -173,12 +173,19 @@ function invertRanges(removals, duration) {
return kept;
}
+// Range bounds are rounded only to shed float noise, never to the millisecond:
+// these values become ffmpeg's -ss/-to, where a millisecond is wide enough to
+// step a boundary onto the next frame at 29.97/30/60 fps.
function finalizeKept(ranges) {
return mergeRanges(ranges)
- .map((range) => ({ start: round3(range.start), end: round3(range.end) }))
+ .map((range) => ({ start: round6(range.start), end: round6(range.end) }))
.filter((range) => round3(range.end - range.start) >= MIN_SEGMENT_SECONDS);
}
+function round6(n) {
+ return Math.round(Number(n) * 1e6) / 1e6;
+}
+
function round3(n) {
return Math.round(Number(n) * 1000) / 1000;
}
diff --git a/skills/media-use/scripts/transcript-cut.mjs b/skills/media-use/scripts/transcript-cut.mjs
index b7e4f468e0..1f232d6ae8 100644
--- a/skills/media-use/scripts/transcript-cut.mjs
+++ b/skills/media-use/scripts/transcript-cut.mjs
@@ -183,11 +183,11 @@ function cutSegment(inputPath, segment, outPath, copy, fade) {
"-y",
"-nostdin",
"-ss",
- fmt(segment.start),
+ seekTime(segment.start),
"-i",
inputPath,
"-to",
- fmt(segment.end - segment.start),
+ seekTime(segment.end - segment.start),
];
if (copy) {
argv.push("-c", "copy", "-avoid_negative_ts", "make_zero");
@@ -265,6 +265,15 @@ function fmt(n) {
.replace(/\.?0+$/, "");
}
+// -ss/-to name frames, and ffmpeg's accurate seek drops everything before the
+// time it is given: at 30 fps, frame 386 sits at 12.866666 s, and a 3-decimal
+// -ss of 12.867 starts that segment on frame 387 instead, leaving the cut a
+// frame short from the join on. So the values ffmpeg gets keep frame precision,
+// while fmt() above stays at 3 decimals for the times that are only printed.
+function seekTime(n) {
+ return Number(n).toFixed(6);
+}
+
function round3(n) {
return Math.round(Number(n) * 1000) / 1000;
}
diff --git a/skills/media-use/scripts/transcript-cut.test.mjs b/skills/media-use/scripts/transcript-cut.test.mjs
index 99b52dce42..59a4d55c96 100644
--- a/skills/media-use/scripts/transcript-cut.test.mjs
+++ b/skills/media-use/scripts/transcript-cut.test.mjs
@@ -115,3 +115,78 @@ test(
);
},
);
+
+// Frame 32 at 30 fps sits at 1.0666666 s. A -ss rounded to 1.067 lands just past
+// it, so ffmpeg's accurate seek starts that segment on frame 33 and the cut comes
+// out a frame short from the join on.
+const THIRTY_FPS_FRAME_32 = 32 / 30;
+
+test("a kept range starting on a 30 fps frame keeps that frame", { skip: !HAS_FFMPEG }, (t) => {
+ const { dir, cleanup } = fixture();
+ t.after(cleanup);
+
+ const source = join(dir, "src.mp4");
+ execFileSync("ffmpeg", [
+ "-y",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=320x180:rate=30",
+ "-f",
+ "lavfi",
+ "-i",
+ `sine=frequency=440:sample_rate=${SAMPLE_RATE}`,
+ "-t",
+ "4",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ "-c:a",
+ "aac",
+ source,
+ ]);
+
+ const transcriptPath = join(dir, "transcript.json");
+ writeFileSync(transcriptPath, JSON.stringify([{ text: "a", start: 0.1, end: 3.9 }]));
+
+ const output = join(dir, "out.mp4");
+ const result = run([
+ "--input",
+ source,
+ "--transcript",
+ transcriptPath,
+ "--keep",
+ `0-1,${THIRTY_FPS_FRAME_32}-3`,
+ "--out",
+ output,
+ "--json",
+ ]);
+ assert.equal(result.status, 0, result.stderr || result.stdout);
+
+ // Frames 0-29 for 0-1s, then frames 32-89 for the second segment. Rounding
+ // -ss to the millisecond drops frame 32 and yields 87.
+ const frames = Number(
+ execFileSync(
+ "ffprobe",
+ [
+ "-v",
+ "error",
+ "-select_streams",
+ "v",
+ "-count_packets",
+ "-show_entries",
+ "stream=nb_read_packets",
+ "-of",
+ "csv=p=0",
+ "--",
+ output,
+ ],
+ { encoding: "utf8" },
+ ).trim(),
+ );
+ assert.equal(frames, 88, `expected 88 frames, got ${frames}`);
+});