Skip to content

Write HTML. Render video. Built for agents.

fix(cli): select containing source frames in snapshots (#4809)

Dante0d3d42320015
diff --git a/packages/cli/src/commands/snapshot.test.ts b/packages/cli/src/commands/snapshot.test.ts
index 76eb8e95e0..4b6789648f 100644
--- a/packages/cli/src/commands/snapshot.test.ts
+++ b/packages/cli/src/commands/snapshot.test.ts
@@ -4,7 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
import { dirname, join } from "node:path";
import { tmpdir } from "node:os";
import { execFileSync } from "node:child_process";
-import { findFFmpeg } from "../browser/ffmpeg.js";
+import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { sourceTimeAt } from "@hyperframes/core";
const snapshotState = vi.hoisted(() => ({
@@ -32,6 +32,7 @@ vi.mock("../utils/staticProjectServer.js", () => ({
import snapshotCommand, {
extractVideoFrameToBuffer,
+ containingSourceFrameIndex,
computeSnapshotTimes,
formatSnapshotTimestamp,
parseZoomScale,
@@ -90,12 +91,8 @@ describe("transparent snapshot capture", () => {
it("pairs every frame with the frame-exact reference frame under --against", () => {
const source = readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8");
expect(source).toContain("against: {");
- expect(source).toContain("extractVideoFrameToBuffer(opts.against, time, false, true)");
+ expect(source).toContain("extractVideoFrameToBuffer(opts.against, time)");
expect(source).toContain('labels: ["render", "reference"]');
- // accurate seek = `-ss` after `-i`, never the keyframe-snap fast path
- expect(source).toContain(
- 'accurateSeek ? ["-i", videoPath, ...seek] : [...seek, "-i", videoPath]',
- );
});
it("resolves and forwards the shared local browser GPU policy", () => {
@@ -347,10 +344,383 @@ describe("resolveSnapshotVideoFrameTime", () => {
});
});
+describe("containingSourceFrameIndex", () => {
+ it.each([
+ [0.4, 9],
+ [22 / 30, 17],
+ [10 / 30, 8],
+ [4.1, 98],
+ ])("selects frame %s on the source presentation grid", (time, expected) => {
+ expect(
+ containingSourceFrameIndex(
+ Array.from({ length: 120 }, (_, n) => n / 24),
+ time,
+ ),
+ ).toBe(expected);
+ });
+ it("keeps exact boundaries despite floating-point round-off", () => {
+ expect(containingSourceFrameIndex([0, 4.1], 4.1 - 4 * Number.EPSILON)).toBe(1);
+ expect(containingSourceFrameIndex([0, 4.1], 4.1 - 0.00001)).toBe(0);
+ });
+ it("uses irregular presentation intervals", () => {
+ expect(containingSourceFrameIndex([0, 0.02, 0.08, 0.15], 0.07)).toBe(1);
+ expect(containingSourceFrameIndex([0, 0.02, 0.08, 0.15], 0.08)).toBe(2);
+ });
+});
+
describe("extractVideoFrameToBuffer", () => {
const ffmpeg = findFFmpeg();
+ const ffprobe = findFFprobe();
+ const noProbe = !ffmpeg || !ffprobe;
+
+ it.skipIf(noProbe)("selects containing frames for snapshots and reference pairs", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-containing-"));
+ try {
+ const clip = join(dir, "clip.mp4");
+ execFileSync(ffmpeg!, [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc=d=2:r=24:s=160x90",
+ "-pix_fmt",
+ "yuv420p",
+ clip,
+ ]);
+ for (const [time, index] of [
+ [0.4, 9],
+ [22 / 30, 17],
+ [10 / 30, 8],
+ [9 / 24, 9],
+ ] as const) {
+ const expected = join(dir, `frame-${index}.png`);
+ execFileSync(ffmpeg!, [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-i",
+ clip,
+ "-vf",
+ `select=eq(n\\,${index})`,
+ "-frames:v",
+ "1",
+ "-y",
+ expected,
+ ]);
+ const actual = await extractVideoFrameToBuffer(clip, time);
+ expect(actual?.equals(readFileSync(expected))).toBe(true);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ it.skipIf(noProbe)("selects a variable-frame-rate presentation interval", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-vfr-"));
+ try {
+ const clip = join(dir, "clip.mp4");
+ execFileSync(ffmpeg!, [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc=d=1:r=10:s=160x90",
+ "-vf",
+ "select=eq(n\\,0)+eq(n\\,1)+eq(n\\,4)+eq(n\\,8)",
+ "-fps_mode",
+ "vfr",
+ "-pix_fmt",
+ "yuv420p",
+ clip,
+ ]);
+ const expected = join(dir, "frame.png");
+ execFileSync(ffmpeg!, [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-i",
+ clip,
+ "-vf",
+ "select=eq(n\\,1)",
+ "-frames:v",
+ "1",
+ "-y",
+ expected,
+ ]);
+ expect((await extractVideoFrameToBuffer(clip, 0.3))?.equals(readFileSync(expected))).toBe(
+ true,
+ );
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ it.skipIf(noProbe).each([
+ ["fragmented MP4", "clip.mp4", ["-movflags", "frag_keyframe+empty_moov"]],
+ ["MPEG-TS", "clip.ts", []],
+ ["MP4", "clip.mp4", []],
+ ["open-GOP MPEG-TS", "clip.ts", ["-x264-params", "open-gop=1"]],
+ ])("matches a full decode past keyframes in %s with B-frames", async (_, name, muxArgs) => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-gop-"));
+ try {
+ const clip = join(dir, name);
+ const quiet = ["-hide_banner", "-loglevel", "error"];
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=160x90", "-pix_fmt", "yuv420p"],
+ ...["-c:v", "libx264", "-bf", "3", "-g", "30", "-keyint_min", "30", "-sc_threshold", "0"],
+ ...muxArgs,
+ clip,
+ ]);
+ // Frames 29 and 59 sit just before a keyframe; 47 and 80 decode from a later keyframe.
+ for (const index of [29, 47, 59, 80]) {
+ const expected = join(dir, `frame-${index}.png`);
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-i", clip, "-vf", `select=eq(n\\,${index})`, "-frames:v", "1", "-y", expected],
+ ]);
+ const actual = await extractVideoFrameToBuffer(clip, (index + 0.5) / 30);
+ expect(actual?.equals(readFileSync(expected)), `frame ${index}`).toBe(true);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ it.skipIf(noProbe)(
+ "leaves an unreadable video blank instead of failing the snapshot",
+ async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-unreadable-"));
+ try {
+ const garbage = join(dir, "garbage.mp4");
+ writeFileSync(garbage, "not a video");
+ expect(await extractVideoFrameToBuffer(join(dir, "missing.mp4"), 1)).toBeNull();
+ expect(await extractVideoFrameToBuffer(garbage, 1)).toBeNull();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
+ );
+
+ it.skipIf(noProbe)(
+ "selects exact frame times in a fragmented MP4 whose start rounds down",
+ async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-start-"));
+ try {
+ // B-frames start this clip at 1024/12288 s, which ffprobe prints as 0.083333.
+ const clip = join(dir, "clip.mp4");
+ const quiet = ["-hide_banner", "-loglevel", "error"];
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-f", "lavfi", "-i", "testsrc2=d=2:r=24:s=160x90", "-pix_fmt", "yuv420p"],
+ ...["-c:v", "libx264", "-bf", "3", "-movflags", "frag_keyframe+empty_moov", clip],
+ ]);
+ for (const index of [0, 1, 12, 47]) {
+ const expected = join(dir, `frame-${index}.png`);
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-i", clip, "-vf", `select=eq(n\\,${index})`, "-frames:v", "1", "-y", expected],
+ ]);
+ const actual = await extractVideoFrameToBuffer(clip, index / 24);
+ expect(actual?.equals(readFileSync(expected)), `frame ${index}`).toBe(true);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
+ );
+
+ it.skipIf(noProbe)(
+ "selects frames in a stream-copy trim whose preroll starts before the video",
+ async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-editlist-"));
+ try {
+ // The trim keeps 1.3 s of preroll packets before the first visible frame (an MP4 edit list).
+ const source = join(dir, "source.mp4");
+ const clip = join(dir, "clip.mp4");
+ const quiet = ["-hide_banner", "-loglevel", "error"];
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-f", "lavfi", "-i", "testsrc2=d=4:r=24:s=160x90", "-pix_fmt", "yuv420p"],
+ ...["-c:v", "libx264", "-bf", "2", source],
+ ]);
+ execFileSync(ffmpeg!, [...quiet, "-ss", "1.3", "-i", source, "-c", "copy", clip]);
+ for (const index of [1, 3, 10, 30]) {
+ const expected = join(dir, `frame-${index}.png`);
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-i", clip, "-vf", `select=eq(n\\,${index})`, "-frames:v", "1", "-y", expected],
+ ]);
+ const actual = await extractVideoFrameToBuffer(clip, index / 24);
+ expect(actual?.equals(readFileSync(expected)), `frame ${index}`).toBe(true);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
+ );
+
+ it.skipIf(noProbe).each([
+ // Audio leads video by 23 ms in a TS mux: frame n sits at 0.0232 + n/30 on the file's clock.
+ [
+ "MPEG-TS with leading audio",
+ "clip.ts",
+ [],
+ [
+ [0, 0],
+ [0.01, 0],
+ [0.05, 0],
+ [0.1, 2],
+ [1, 29],
+ ],
+ ],
+ // Video starts 0.5 s after the audio, so earlier times hold the first frame.
+ [
+ "MP4 with late video",
+ "clip.mp4",
+ ["-vf", "setpts=PTS+0.5/TB", "-fps_mode", "passthrough"],
+ [
+ [0, 0],
+ [0.1, 0],
+ [0.55, 1],
+ [1, 15],
+ ],
+ ],
+ ] as const)("measures frame times on the file clock in %s", async (_, name, videoArgs, cases) => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-av-"));
+ try {
+ const clip = join(dir, name);
+ const quiet = ["-hide_banner", "-loglevel", "error"];
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-f", "lavfi", "-i", "testsrc2=d=2:r=30:s=160x90", "-f", "lavfi", "-i", "sine=d=3"],
+ ...videoArgs,
+ ...["-c:v", "libx264", "-bf", "3", "-pix_fmt", "yuv420p", "-c:a", "aac", clip],
+ ]);
+ for (const [time, index] of cases) {
+ const expected = join(dir, `frame-${index}.png`);
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-i", clip, "-map", "0:v:0", "-vf", `select=eq(n\\,${index})`],
+ ...["-frames:v", "1", "-y", expected],
+ ]);
+ const actual = await extractVideoFrameToBuffer(clip, time);
+ expect(actual?.equals(readFileSync(expected)), `t=${time}`).toBe(true);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ it.skipIf(noProbe)("selects frames in a TS whose timestamps start below zero", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-negative-"));
+ try {
+ const clip = join(dir, "clip.ts");
+ const quiet = ["-hide_banner", "-loglevel", "error"];
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=160x90", "-c:v", "libx264", "-g", "1"],
+ ...[
+ "-output_ts_offset",
+ "-2",
+ "-avoid_negative_ts",
+ "disabled",
+ "-mpegts_copyts",
+ "1",
+ clip,
+ ],
+ ]);
+ const frameAt = async (time: number, index: number) => {
+ const expected = join(dir, `frame-${index}.png`);
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-i", clip, "-vf", `select=eq(n\\,${index})`, "-frames:v", "1", "-y", expected],
+ ]);
+ const actual = await extractVideoFrameToBuffer(clip, time);
+ expect(actual?.equals(readFileSync(expected)), `t=${time}`).toBe(true);
+ };
+ for (const [time, index] of [
+ [0, 0],
+ [0.05, 1],
+ [0.1, 3],
+ [1, 30],
+ [2.5, 75],
+ ] as const) {
+ await frameAt(time, index);
+ }
+ // Without ffprobe, the plain seek keeps main's first-frame-at-or-after behaviour.
+ vi.stubEnv("HYPERFRAMES_FFPROBE_PATH", join(dir, "missing-ffprobe"));
+ for (const [time, index] of [
+ [0, 0],
+ [0.05, 2],
+ [1, 30],
+ ] as const)
+ await frameAt(time, index);
+ } finally {
+ vi.unstubAllEnvs();
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ it.skipIf(noProbe)("selects frames in a TS with frequent open-GOP keyframes", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-open-gop-"));
+ try {
+ // A seek to 0.5 s lands on the keyframe at 1 s here, so the frames between are never decoded.
+ const clip = join(dir, "clip.ts");
+ const quiet = ["-hide_banner", "-loglevel", "error"];
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=160x90", "-c:v", "libx264", "-g", "15"],
+ ...["-bf", "3", "-x264-params", "open-gop=1:scenecut=0", clip],
+ ]);
+ for (const [time, index] of [
+ [0.5, 15],
+ [0.51, 15],
+ [0.99, 29],
+ [1.5, 45],
+ [1.99, 59],
+ ] as const) {
+ const expected = join(dir, `frame-${index}.png`);
+ execFileSync(ffmpeg!, [
+ ...quiet,
+ ...["-i", clip, "-vf", `select=eq(n\\,${index})`, "-frames:v", "1", "-y", expected],
+ ]);
+ const actual = await extractVideoFrameToBuffer(clip, time);
+ expect(actual?.equals(readFileSync(expected)), `t=${time}`).toBe(true);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ it.skipIf(!ffmpeg)("extracts without ffprobe, as before frame selection needed it", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-no-ffprobe-"));
+ vi.stubEnv("HYPERFRAMES_FFPROBE_PATH", join(dir, "missing-ffprobe"));
+ try {
+ const clip = join(dir, "clip.mp4");
+ execFileSync(ffmpeg!, [
+ ...["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc=d=1:r=24:s=160x90"],
+ ...["-pix_fmt", "yuv420p", clip],
+ ]);
+ const expected = join(dir, "frame.png");
+ execFileSync(ffmpeg!, [
+ ...["-hide_banner", "-loglevel", "error", "-i", clip, "-vf", "select=eq(n\\,12)"],
+ ...["-frames:v", "1", "-y", expected],
+ ]);
+ expect((await extractVideoFrameToBuffer(clip, 0.5))?.equals(readFileSync(expected))).toBe(
+ true,
+ );
+ } finally {
+ vi.unstubAllEnvs();
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
- it.skipIf(!ffmpeg)(
+ it.skipIf(noProbe)(
"gives a 24 fps clip's real last frame for a held tail that lands past it",
async () => {
const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-tail-"));
@@ -359,10 +729,10 @@ describe("extractVideoFrameToBuffer", () => {
const source = ["-f", "lavfi", "-i", "testsrc=d=1:r=24:s=160x90", "-pix_fmt", "yuv420p"];
execFileSync(ffmpeg!, ["-hide_banner", "-loglevel", "error", ...source, clip]);
- const held = await extractVideoFrameToBuffer(clip, 1 - 1 / 30, false, false, true);
- const lastFrame = await extractVideoFrameToBuffer(clip, 23 / 24, false, true);
+ const held = await extractVideoFrameToBuffer(clip, 1 - 1 / 30, false, true);
+ const lastFrame = await extractVideoFrameToBuffer(clip, 23 / 24);
- expect(await extractVideoFrameToBuffer(clip, 1 - 1 / 30)).toBeNull();
+ expect((await extractVideoFrameToBuffer(clip, 1 - 1 / 30))?.equals(lastFrame!)).toBe(true);
expect(lastFrame).not.toBeNull();
expect(held?.equals(lastFrame!)).toBe(true);
} finally {
diff --git a/packages/cli/src/commands/snapshot.ts b/packages/cli/src/commands/snapshot.ts
index 6da99583b8..db733a5c0b 100644
--- a/packages/cli/src/commands/snapshot.ts
+++ b/packages/cli/src/commands/snapshot.ts
@@ -32,7 +32,8 @@ import { formatLintFindings } from "../utils/lintFormat.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { serveStaticProjectHtml } from "../utils/staticProjectServer.js";
import { c } from "../ui/colors.js";
-import { findFFmpeg, getFFmpegInstallHint } from "../browser/ffmpeg.js";
+import { runCancellableProcess } from "../utils/cancellableProcess.js";
+import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
import { parseAngle, type Camera } from "./motionShotLayout.js";
import type { Example } from "./_examples.js";
import { loadOptionalPackage } from "../utils/optionalPackages.js";
@@ -176,51 +177,167 @@ export async function recaptureSnapshotComposite(page: SnapshotCompositePage): P
}
}
-/**
- * Extract a single frame from a video file at `timeSeconds` via FFmpeg.
- * Used to work around Chrome-headless's inability to reliably seek
- * <video> elements during snapshot capture.
- */
+export function containingSourceFrameIndex(timestamps: readonly number[], time: number): number {
+ if (!Number.isFinite(time)) return -1;
+ const tolerance = 16 * Number.EPSILON * Math.max(1, Math.abs(time));
+ let index = -1;
+ for (const [i, timestamp] of timestamps.entries()) {
+ if (timestamp > time + tolerance) break;
+ index = i;
+ }
+ return index;
+}
+
+/** Keyframe seek plus the decoded-time threshold that selects the frame containing `time`, or null. */
+async function probeContainingSourceFrameSeek(
+ videoPath: string,
+ time: number,
+ holdLastFrame: boolean,
+): Promise<{ seek?: number; select?: number } | null> {
+ const ffprobe = findFFprobe();
+ // Without ffprobe, fall back to a plain seek: the first frame at or after `time`.
+ if (!ffprobe) return { seek: time };
+ // Packet timestamps need no decode, so a late sample costs a demux, not a decode from the start.
+ const probe = async (
+ readIntervals: string,
+ entries: string,
+ ): Promise<Record<string, unknown> | null> => {
+ try {
+ const { stdout } = await runCancellableProcess(
+ ffprobe,
+ [
+ ...["-v", "error", "-select_streams", "v:0", "-read_intervals", readIntervals],
+ ...["-show_entries", entries, "-of", "json", "--", videoPath],
+ ],
+ { timeoutMs: FFMPEG_EXTRACT_TIMEOUT_MS, maxBufferBytes: 32 * 1024 * 1024 },
+ );
+ const parsed: unknown = JSON.parse(stdout);
+ return typeof parsed === "object" && parsed !== null
+ ? (parsed as Record<string, unknown>)
+ : null;
+ } catch {
+ // An unreadable source blanks this one video, as a failed ffmpeg extraction does.
+ return null;
+ }
+ };
+ const data = await probe("%+#1", "stream=time_base,start_pts:format=start_time,duration");
+ if (data === null || !Array.isArray(data.streams)) return null;
+ const stream: unknown = data.streams[0];
+ if (
+ typeof stream !== "object" ||
+ stream === null ||
+ !("time_base" in stream) ||
+ typeof stream.time_base !== "string"
+ )
+ return null;
+ const [numerator, denominator] = stream.time_base.split("/");
+ const timeBase = Number(numerator) / Number(denominator);
+ if (!Number.isFinite(timeBase) || timeBase <= 0) return null;
+ const format = "format" in data ? data.format : null;
+ const start =
+ typeof format === "object" && format !== null && "start_time" in format
+ ? Number(format.start_time)
+ : 0;
+ const duration =
+ typeof format === "object" && format !== null && "duration" in format
+ ? Number(format.duration)
+ : NaN;
+ if (!holdLastFrame && Number.isFinite(duration) && time >= duration) return null;
+ const formatStart = Number.isFinite(start) ? start : 0;
+ // format.start_time is rounded to microseconds; when the video sets it, use its exact start_pts.
+ const startPts =
+ "start_pts" in stream && typeof stream.start_pts === "number" ? stream.start_pts : NaN;
+ const startTicks =
+ Math.abs(startPts * timeBase - formatStart) <= 1e-6 ? startPts : formatStart / timeBase;
+ // An absolute end: edit-list preroll packets start before the video, so a relative window can stop short.
+ const packetData = await probe(`%${startTicks * timeBase + time + 1}`, "packet=pts,dts,flags");
+ if (packetData === null || !Array.isArray(packetData.packets)) return null;
+ const packets: { pts: number; dts: number; key: boolean }[] = [];
+ for (const packet of packetData.packets) {
+ if (typeof packet !== "object" || packet === null) return null;
+ const pts = "pts" in packet ? packet.pts : "dts" in packet ? packet.dts : undefined;
+ if (typeof pts !== "number") return null;
+ const dts = "dts" in packet && typeof packet.dts === "number" ? packet.dts : pts;
+ const key = "flags" in packet && typeof packet.flags === "string" && packet.flags.includes("K");
+ packets.push({ pts: (pts - startTicks) * timeBase, dts: (dts - startTicks) * timeBase, key });
+ }
+ // Packets arrive in decode order; B-frames make that differ from presentation order.
+ const timestamps = packets.map((packet) => packet.pts).sort((a, b) => a - b);
+ // Before the video's first frame (audio leads it), hold that frame, as the engine does.
+ const index = Math.max(0, containingSourceFrameIndex(timestamps, time));
+ if (timestamps.length === 0 || !Number.isFinite(time)) return null;
+ const frame = timestamps[index]!;
+ const previous = timestamps[index - 1] ?? frame - 0.001;
+ const mid = (previous + frame) / 2;
+ // Demuxers land on a keyframe of their choosing (MPEG-TS often a later one), so observe where a
+ // seek lands; a seek to exactly that keyframe trims its leading frames, so it is `start_t`.
+ const ffmpeg = findFFmpeg();
+ const landing = async (seek: number) => {
+ const raw = ffmpeg ? await probeSeekLanding(ffmpeg, videoPath, seek) : null;
+ return raw === null ? null : raw - startTicks * timeBase;
+ };
+ const candidates = packets.filter(
+ (packet) => packet.key && packet.pts > 0 && packet.pts <= frame,
+ );
+ for (const candidate of candidates.reverse().slice(0, 3)) {
+ const landed = await landing(candidate.pts);
+ if (landed === null) break;
+ if (landed <= 0 || landed > frame) continue;
+ const again = Math.abs(landed - candidate.pts) < 1e-6 ? landed : await landing(landed);
+ if (again !== null && Math.abs(again - landed) < 1e-6)
+ return { seek: landed, select: mid - landed };
+ }
+ // No seek: the first frame FFmpeg emits is the first one at or after the file start.
+ return { select: mid - (timestamps.find((t) => t >= 0) ?? frame) };
+}
+
+/** Raw stream time of the packet an input seek to `seek` lands on, read without decoding. */
+async function probeSeekLanding(
+ ffmpeg: string,
+ videoPath: string,
+ seek: number,
+): Promise<number | null> {
+ try {
+ const { stdout } = await runCancellableProcess(
+ ffmpeg,
+ [
+ ...["-v", "error", "-copyts", "-noaccurate_seek", "-ss", String(seek), "-i", videoPath],
+ ...["-map", "0:v:0", "-c", "copy", "-frames:v", "1", "-f", "framecrc", "-"],
+ ],
+ { timeoutMs: FFMPEG_EXTRACT_TIMEOUT_MS, maxBufferBytes: 1024 * 1024 },
+ );
+ const base = /^#tb 0: (\d+)\/(\d+)/m.exec(stdout);
+ const row = /^0,\s*-?\d+,\s*(-?\d+),/m.exec(stdout);
+ if (!base || !row) return null;
+ return (Number(row[1]) * Number(base[1])) / Number(base[2]);
+ } catch {
+ return null;
+ }
+}
+
export async function extractVideoFrameToBuffer(
videoPath: string,
timeSeconds: number,
useVp9AlphaDecoder = false,
- accurateSeek = false,
holdLastFrame = false,
): Promise<Buffer | null> {
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
const outPath = join(tmp, "frame.png");
try {
const ffmpegPath = requireSnapshotFfmpeg(findFFmpeg());
- // `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
- // (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
- // `accurateSeek` puts `-ss` after `-i` (decode from the start) for frame-exact
- // reference pairs, where ±1 frame would read as a real mismatch.
- const args = ["-hide_banner", "-loglevel", "error"];
- if (useVp9AlphaDecoder) {
- args.push("-c:v", "libvpx-vp9");
- }
- const decoderArgs = [...args];
- const seek = ["-ss", String(Math.max(0, timeSeconds))];
- args.push(
- ...(accurateSeek ? ["-i", videoPath, ...seek] : [...seek, "-i", videoPath]),
- "-frames:v",
- "1",
- "-q:v",
- "2",
- "-y",
- outPath,
+ const seek = await probeContainingSourceFrameSeek(
+ videoPath,
+ Math.max(0, timeSeconds),
+ holdLastFrame,
);
- let result = await runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
- if (holdLastFrame && result.code === 0 && !result.timedOut && !existsSync(outPath)) {
- // Past the last frame's timestamp FFmpeg writes nothing; decode the final second and keep its last frame.
- const tail = ["-sseof", "-1", "-i", videoPath, "-update", "1", "-q:v", "2", "-y", outPath];
- result = await runFfmpegOnce(
- ffmpegPath,
- [...decoderArgs, ...tail],
- FFMPEG_EXTRACT_TIMEOUT_MS,
- );
- }
+ if (seek === null) return null;
+ const args = ["-hide_banner", "-loglevel", "error"];
+ if (useVp9AlphaDecoder) args.push("-c:v", "libvpx-vp9");
+ if (seek.seek !== undefined) args.push("-ss", String(seek.seek));
+ args.push("-i", videoPath, "-map", "0:v:0");
+ if (seek.select !== undefined) args.push("-vf", `select=gte(t-start_t\\,${seek.select})`);
+ args.push("-frames:v", "1", "-q:v", "2", "-y", outPath);
+ const result = await runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
if (result.code !== 0 || result.timedOut || !existsSync(outPath)) return null;
return readFileSync(outPath);
} finally {
@@ -599,7 +716,6 @@ async function captureSnapshots(
ffmpegInput,
Math.max(0, v.relTime),
useVp9AlphaDecoder,
- false,
v.srcDuration > 0 && v.relTime >= v.srcDuration - 1,
);
if (!png) continue;
@@ -665,7 +781,7 @@ async function captureSnapshots(
if (opts.against) {
// Frame-exact reference frame beside the render, so a rebuild can be
// checked against its footage without hand-rolled ffmpeg + montage.
- const refPng = await extractVideoFrameToBuffer(opts.against, time, false, true);
+ const refPng = await extractVideoFrameToBuffer(opts.against, time);
if (!refPng) {
console.error(
` ${c.warn("⚠")} --against has no frame at ${timeLabel} — reference pair skipped`,