Skip to content

Write HTML. Render video. Built for agents.

perf(cli): snapshot decodes phone video frames at their on-screen size, several at once (#5917)

Miguel Ángel122489c34dab
diff --git a/packages/cli/src/commands/snapshot.test.ts b/packages/cli/src/commands/snapshot.test.ts
index 4da4a51b40..130de77f0a 100644
--- a/packages/cli/src/commands/snapshot.test.ts
+++ b/packages/cli/src/commands/snapshot.test.ts
@@ -742,6 +742,40 @@ describe("extractVideoFrameToBuffer", () => {
}
},
);
+
+ it.skipIf(noProbe)(
+ "extracts a frame at its displayed size, never larger than the source",
+ async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-displayed-"));
+ try {
+ const size = async (clip: string, displayedPixels: number) => {
+ const png = await extractVideoFrameToBuffer(clip, 0.5, false, false, displayedPixels);
+ if (!png) throw new Error(`no frame from ${clip}`);
+ return [png.readUInt32BE(16), png.readUInt32BE(20)];
+ };
+ for (const { name, width, height } of [
+ { name: "landscape", width: 160, height: 90 },
+ { name: "portrait", width: 90, height: 160 },
+ ]) {
+ const clip = join(dir, `${name}.mp4`);
+ const input = [
+ "-f",
+ "lavfi",
+ "-i",
+ `testsrc=d=1:r=24:s=${width}x${height}`,
+ "-pix_fmt",
+ "yuv420p",
+ ];
+ execFileSync(ffmpeg!, ["-hide_banner", "-loglevel", "error", ...input, clip]);
+ expect(await size(clip, 0)).toEqual([width, height]);
+ expect(await size(clip, 1000)).toEqual([width, height]);
+ expect(Math.min(...(await size(clip, 32)))).toBe(32);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
+ );
});
describe("resolveSnapshotVideoClipStart", () => {
diff --git a/packages/cli/src/commands/snapshot.ts b/packages/cli/src/commands/snapshot.ts
index 7bd940c2c4..bbd026d426 100644
--- a/packages/cli/src/commands/snapshot.ts
+++ b/packages/cli/src/commands/snapshot.ts
@@ -17,6 +17,7 @@ import {
type ArtifactCapture,
} from "@hyperframes/studio-server/history";
import { pngDimensions } from "../utils/pngDimensions.js";
+import { runWithConcurrencyLimit } from "../utils/concurrency.js";
import { failCommand } from "../utils/commandResult.js";
// fallow-ignore-file complexity
import { defineCommand } from "citty";
@@ -97,6 +98,8 @@ function orbitStageSource(): string {
* clip (corrupt media, stalled network mount, codec edge case) cannot wedge
* `hyperframes snapshot` indefinitely. */
const FFMPEG_EXTRACT_TIMEOUT_MS = 30_000;
+/** Each 4K decode is already multi-threaded, so a few at once fill the cores without exhausting memory. */
+const VIDEO_FRAME_EXTRACT_CONCURRENCY = 4;
/** Keep millisecond-level snapshot timing proof without leaking floating-point noise. */
export function formatSnapshotTimestamp(time: number): string {
@@ -355,6 +358,7 @@ export async function extractVideoFrameToBuffer(
timeSeconds: number,
useVp9AlphaDecoder = false,
holdLastFrame = false,
+ displayedPixels = 0,
): Promise<Buffer | null> {
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
const outPath = join(tmp, "frame.png");
@@ -370,7 +374,15 @@ export async function extractVideoFrameToBuffer(
if (useVp9AlphaDecoder) args.push("-c:v", "libvpx-vp9");
if (seek.seek !== undefined) args.push("-ss", String(seek.seek));
args.push("-i", videoPath, "-map", "0:v:0");
- if (seek.select !== undefined) args.push("-vf", `select=gte(t-start_t\\,${seek.select})`);
+ const filters: string[] = [];
+ if (seek.select !== undefined) filters.push(`select=gte(t-start_t\\,${seek.select})`);
+ // Shrink so the short side still covers the displayed box at any rotation or fit; never enlarge.
+ if (displayedPixels > 0) {
+ filters.push(
+ `scale='min(iw,${displayedPixels})':'min(ih,${displayedPixels})':force_original_aspect_ratio=increase`,
+ );
+ }
+ if (filters.length > 0) args.push("-vf", filters.join(","));
args.push("-frames:v", "1", "-q:v", "2", "-y", outPath);
const result = await runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
if (result.code !== 0 || result.timedOut || !existsSync(outPath)) return null;
@@ -708,6 +720,7 @@ async function captureSnapshots(
};
return Array.from(document.querySelectorAll("video")).map((el) => {
const v = el as HTMLVideoElement;
+ const box = v.getBoundingClientRect();
const authoredStart = parseFloat(v.dataset.start ?? "0") || 0;
const runtimeResolvedStart = runtimeWindow.__hfResolveMediaStartSeconds?.(v);
const mediaStart =
@@ -730,6 +743,8 @@ async function captureSnapshots(
srcDuration: srcDur,
mediaStart,
loop: v.loop,
+ displayedSize: Math.max(box.width, box.height),
+ devicePixelRatio: window.devicePixelRatio,
};
});
});
@@ -769,51 +784,49 @@ async function captureSnapshots(
: [{ ...candidate, start, playbackRate, duration, relTime: frameTime }];
});
- const updates: Array<{ videoId: string; dataUri: string }> = [];
- for (const v of active) {
- // Resolve the <video> src to an FFmpeg input. Prefer a project-local
- // file (fast, sandboxed); fall back to the absolute http(s) URL for
- // remote assets (e.g. an S3-hosted clip embedded by an upstream agent)
- // — FFmpeg reads http(s) input directly, and Chrome-headless can't seek
- // it either, so without this those videos render blank in snapshots.
- let ffmpegInput: string | null = null;
- let inputIsLocal = false;
- try {
- const url = new URL(v.src);
- const decodedPath = decodeURIComponent(url.pathname).replace(/^\//, "");
- const candidate = resolve(projectDir, decodedPath);
- const rel = relative(projectDir, candidate);
- if (!rel.startsWith("..") && !isAbsolute(rel) && existsSync(candidate)) {
- ffmpegInput = candidate;
- inputIsLocal = true;
- } else if (url.protocol === "http:" || url.protocol === "https:") {
- ffmpegInput = url.href;
+ const extracted = await runWithConcurrencyLimit(
+ active,
+ VIDEO_FRAME_EXTRACT_CONCURRENCY,
+ async (v) => {
+ // Prefer the project-local file; remote http(s) clips go to FFmpeg directly,
+ // since headless Chrome can't seek them and they would render blank.
+ let ffmpegInput: string | null = null;
+ let inputIsLocal = false;
+ try {
+ const url = new URL(v.src);
+ const decodedPath = decodeURIComponent(url.pathname).replace(/^\//, "");
+ const candidate = resolve(projectDir, decodedPath);
+ const rel = relative(projectDir, candidate);
+ if (!rel.startsWith("..") && !isAbsolute(rel) && existsSync(candidate)) {
+ ffmpegInput = candidate;
+ inputIsLocal = true;
+ } else if (url.protocol === "http:" || url.protocol === "https:") {
+ ffmpegInput = url.href;
+ }
+ } catch {
+ /* unresolvable src (e.g. blob:, data:) — skip */
}
- } catch {
- /* unresolvable src (e.g. blob:, data:) — skip */
- }
- if (!ffmpegInput) continue;
- // VP9-alpha detection shells out to ffprobe, which has no timeout.
- // Only probe local files (filesystem-bounded); for remote URLs skip it
- // (pass false) so a stalled host can't wedge snapshot in ffprobe before
- // the bounded extractVideoFrameToBuffer below ever runs. Remote
- // VP9-alpha overlays aren't a current path — revisit with a bounded
- // ffprobe if one appears.
- const useVp9AlphaDecoder = inputIsLocal
- ? await shouldUseVp9AlphaDecoder(ffmpegInput)
- : false;
- const png = await extractVideoFrameToBuffer(
- ffmpegInput,
- Math.max(0, v.relTime),
- useVp9AlphaDecoder,
- v.srcDuration > 0 && v.relTime >= v.srcDuration - 1,
- );
- if (!png) continue;
- updates.push({
- videoId: v.id,
- dataUri: `data:image/png;base64,${png.toString("base64")}`,
- });
- }
+ if (!ffmpegInput) return null;
+ // The VP9-alpha probe has no timeout, so only local files get it;
+ // a stalled remote host must not wedge snapshot before the bounded extract.
+ const useVp9AlphaDecoder = inputIsLocal
+ ? await shouldUseVp9AlphaDecoder(ffmpegInput)
+ : false;
+ const png = await extractVideoFrameToBuffer(
+ ffmpegInput,
+ Math.max(0, v.relTime),
+ useVp9AlphaDecoder,
+ v.srcDuration > 0 && v.relTime >= v.srcDuration - 1,
+ Math.ceil(
+ v.displayedSize *
+ (opts.zoom ? (opts.zoomScale ?? DEFAULT_ZOOM_SCALE) : v.devicePixelRatio),
+ ),
+ );
+ if (!png) return null;
+ return { videoId: v.id, dataUri: `data:image/png;base64,${png.toString("base64")}` };
+ },
+ );
+ const updates = extracted.filter((update) => update !== null);
if (active.length > 0 && updates.length < active.length) {
const missed = active.length - updates.length;