Skip to content

Write HTML. Render video. Built for agents.

perf(producer): cloud render chunks hand Chrome each video near the size it is shown (#6001)

Miguel Ángel7cc5dd382cd0
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 5559c948eb..26fd4571d7 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -434,6 +434,8 @@ export {
showVideoElements,
queryVideoElementBounds,
queryElementStacking,
+ measureVideoShownBoxes,
+ type ShownBox,
type VideoElementBounds,
type ElementStackingInfo,
} from "./services/videoFrameInjector.js";
diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts
index bd3f9a6e6e..442e77035a 100644
--- a/packages/engine/src/services/videoFrameExtractor.test.ts
+++ b/packages/engine/src/services/videoFrameExtractor.test.ts
@@ -2238,6 +2238,38 @@ describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
},
60_000,
);
+
+ it("halves frames while they still cover the box a video is shown in", async () => {
+ const firstFrameSize = async (
+ videoId: string,
+ shownBox: { width: number; height: number },
+ ) => {
+ const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, {
+ fps: 30,
+ outputDir: FIXTURE_DIR,
+ format: "jpg",
+ frameRanges: [{ firstFrame: 0, frames: 1 }],
+ shownBox,
+ });
+ const [frame] = result.framePaths.values();
+ return spawnSync("ffprobe", [
+ "-v",
+ "error",
+ "-show_entries",
+ "stream=width,height",
+ "-of",
+ "csv=p=0",
+ "--",
+ frame!,
+ ])
+ .stdout.toString()
+ .trim();
+ };
+ // 64x36 source: halved as often as the box allows (even sides round up); never below the box.
+ expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10");
+ expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18");
+ expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36");
+ }, 30_000);
});
});
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index 2705cbcc26..989705f8d1 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -253,6 +253,7 @@ export interface ExtractionOptions {
/** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
contentKeyedDirs?: readonly string[];
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
+ shownBox?: { width: number; height: number };
}
const DOWNLOAD_SUBDIR = "_downloads";
@@ -263,6 +264,14 @@ const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
+function halveToCoverScaleFilter(box: { width: number; height: number }): string {
+ const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`;
+ const halvings = `floor(log(1/${fit})/log(2))`;
+ const side = (dim: "iw" | "ih") =>
+ `if(lt(${fit}\\,0.5)\\,2*ceil(${dim}/pow(2\\,${halvings})/2)\\,${dim})`;
+ return `scale=w=${side("iw")}:h=${side("ih")}:flags=area+accurate_rnd`;
+}
+
const SDR_JPEG_AS_BT601_FULL_RANGE_FILTER =
"scale=flags=neighbor+accurate_rnd,format=gbrp,scale=out_color_matrix=bt601:out_range=pc:flags=neighbor,format=yuv420p";
@@ -1094,7 +1103,10 @@ async function extractVideoFramesRangeUnredacted(
...(isHdr ? [] : chromeGuessForUntaggedMatrix(metadata)),
SDR_CANVAS_PASSTHROUGH_FILTER,
);
- if (format === "jpg") vfFilters.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER);
+ if (format === "jpg") {
+ if (options.shownBox) vfFilters.push(halveToCoverScaleFilter(options.shownBox));
+ vfFilters.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER);
+ }
}
const encodeArgs = ["-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"];
// Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.
diff --git a/packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts b/packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts
new file mode 100644
index 0000000000..0236ace8d8
--- /dev/null
+++ b/packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts
@@ -0,0 +1,59 @@
+import { spawnSync } from "node:child_process";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { Browser, Page } from "puppeteer-core";
+import { resolveHeadlessShellPath } from "./browserManager.js";
+import { measureVideoShownBoxes } from "./videoFrameInjector.js";
+import { closeTestChrome, launchTestChrome } from "./__fixtures__/loopback.js";
+
+const chromePath = resolveHeadlessShellPath();
+const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
+
+// `#zoom` grows from 400x225 at t=0 to 800x450 at t=1; `#late` is only laid out from t=0.5.
+const page1 = `<body style="margin:0">
+ <video id="zoom" style="position:absolute;width:400px;height:225px;transform-origin:0 0"></video>
+ <video id="late" style="position:absolute;top:500px;width:320px;height:180px;display:none"></video>
+ <video id="graded" data-color-grading="warm" style="width:200px;height:100px"></video>
+ <video id="natural" style="width:200px;height:100px;object-fit:none"></video>
+ <script>
+ window.__hf = { seek(t) {
+ document.getElementById("zoom").style.transform = "scale(" + (1 + t) + ")";
+ document.getElementById("late").style.display = t >= 0.5 ? "block" : "none";
+ } };
+ </script>
+</body>`;
+
+describe.skipIf(!hasBrowser)("measureVideoShownBoxes in Chrome", () => {
+ let browser: Browser;
+
+ beforeAll(async () => {
+ browser = await launchTestChrome(chromePath);
+ });
+ afterAll(() => closeTestChrome(browser));
+
+ async function open(html: string, deviceScaleFactor: number): Promise<Page> {
+ const page = await browser.newPage();
+ await page.setViewport({ width: 1000, height: 800, deviceScaleFactor });
+ await page.setContent(html);
+ return page;
+ }
+
+ it("returns each video's largest box over the seeked times, in device pixels", async () => {
+ const page = await open(page1, 2);
+ const ids = ["zoom", "late", "graded", "natural", "missing"];
+ const boxes = await measureVideoShownBoxes(page, ids, [0, 0.25, 1]);
+ expect(Object.fromEntries(boxes)).toEqual({
+ zoom: { width: 1600, height: 900 },
+ late: { width: 640, height: 360 },
+ });
+ await page.close();
+ });
+
+ it("keeps every video full size when the page has a canvas that could sample it", async () => {
+ const page = await open(
+ page1.replace("<script>", '<canvas width="10" height="10"></canvas><script>'),
+ 1,
+ );
+ expect((await measureVideoShownBoxes(page, ["zoom", "late"], [0, 1])).size).toBe(0);
+ await page.close();
+ });
+});
diff --git a/packages/engine/src/services/videoFrameInjector.ts b/packages/engine/src/services/videoFrameInjector.ts
index f6457e03e6..be78081dc3 100644
--- a/packages/engine/src/services/videoFrameInjector.ts
+++ b/packages/engine/src/services/videoFrameInjector.ts
@@ -15,6 +15,7 @@ import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotSe
import { type BeforeCaptureHook } from "./frameCapture.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import {
+ HF_COLOR_GRADING_ATTR,
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
RENDER_FRAME_ID_PREFIX,
RENDER_FRAME_ID_SUFFIX,
@@ -342,6 +343,74 @@ async function setVideoElementsVisibility(
);
}
+export interface ShownBox {
+ width: number;
+ height: number;
+}
+
+/**
+ * Seeks to each of `times`; returns each video's largest drawn box in device pixels. Videos that are
+ * graded, sampled by a canvas, `object-fit: none | scale-down` or never laid out get none (full size).
+ */
+export async function measureVideoShownBoxes(
+ page: Page,
+ videoIds: readonly string[],
+ times: readonly number[],
+): Promise<Map<string, ShownBox>> {
+ if (videoIds.length === 0 || times.length === 0) return new Map();
+ const boxes = await page.evaluate(
+ async (
+ ids: string[],
+ seekTimes: number[],
+ gradingAttr: string,
+ gradingCanvasPrefix: string,
+ ) => {
+ const hf = (window as Window & { __hf?: { seek?: (t: number) => void } }).__hf;
+ if (typeof hf?.seek !== "function") return [];
+ const waitForSeek = (window as Window & { __hfWaitForSeekCompletion?: () => Promise<void> })
+ .__hfWaitForSeekCompletion;
+ const largest = new Map<string, ShownBox>();
+ const fullSize = new Set<string>();
+ const keepsFullSize = (video: Element): boolean => {
+ if (video.hasAttribute(gradingAttr)) return true;
+ const fit = window.getComputedStyle(video).objectFit;
+ return fit === "none" || fit === "scale-down";
+ };
+ const measure = (id: string): void => {
+ const video = window.__hfMediaEl?.(id) ?? document.getElementById(id);
+ if (!video || keepsFullSize(video)) {
+ fullSize.add(id);
+ return;
+ }
+ const rect = video.getBoundingClientRect();
+ const box = largest.get(id) ?? { width: 0, height: 0 };
+ box.width = Math.max(box.width, rect.width * window.devicePixelRatio);
+ box.height = Math.max(box.height, rect.height * window.devicePixelRatio);
+ largest.set(id, box);
+ };
+ const pageHasCanvas = (): boolean =>
+ Array.from(document.querySelectorAll("canvas")).some(
+ (canvas) => !canvas.id.startsWith(gradingCanvasPrefix),
+ );
+ for (const t of seekTimes) {
+ hf.seek(t);
+ await waitForSeek?.();
+ // A canvas could sample any video, so every video keeps full size.
+ if (pageHasCanvas()) return [];
+ for (const id of ids) measure(id);
+ }
+ return Array.from(largest).filter(
+ ([id, box]) => !fullSize.has(id) && box.width > 0 && box.height > 0,
+ );
+ },
+ [...videoIds],
+ [...times],
+ HF_COLOR_GRADING_ATTR,
+ HF_COLOR_GRADING_CANVAS_ID_PREFIX,
+ );
+ return new Map(boxes);
+}
+
/**
* Query the current bounds, transform, and visibility of video elements.
* Called after seeking (so GSAP has moved things) but before the screenshot.
diff --git a/packages/producer/src/services/distributed/chunkFrames.ts b/packages/producer/src/services/distributed/chunkFrames.ts
index b065958a62..82ef686af1 100644
--- a/packages/producer/src/services/distributed/chunkFrames.ts
+++ b/packages/producer/src/services/distributed/chunkFrames.ts
@@ -13,6 +13,7 @@ import {
extractVideoFramesRange,
type FrameRange,
type FrameLookupTable,
+ type ShownBox,
} from "@hyperframes/engine";
import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";
@@ -167,6 +168,34 @@ export function buildChunkCaptureOptions(input: {
}
/** Extracts the given frames of deferred videos under the names a full extraction gives them. */
+export function shownDeferredFrameIndexes(
+ planVideos: PlanVideosJson,
+ frames: Iterable<number>,
+ fps: { num: number; den: number },
+): Map<string, Set<number>> {
+ const shown = new Map<string, Set<number>>();
+ const deferred = planVideos.extracted.filter((video) => video.deferredRange);
+ if (deferred.length === 0) return shown;
+ const table = createFrameLookupTable(
+ planVideos.videos,
+ deferred.map((video) => ({
+ ...video,
+ outputDir: "",
+ framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
+ })),
+ resolveRenderFpsConfig(fps).value,
+ );
+ for (const frame of frames) {
+ const globalTime = (frame * fps.den) / fps.num;
+ for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
+ const indexes = shown.get(videoId) ?? new Set<number>();
+ indexes.add(payload.frameIndex);
+ shown.set(videoId, indexes);
+ }
+ }
+ return shown;
+}
+
export async function extractDeferredVideoFramesForChunk(input: {
planDir: string;
planVideos: PlanVideosJson;
@@ -175,6 +204,7 @@ export async function extractDeferredVideoFramesForChunk(input: {
outputRoot: string;
cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
remoteSources?: PlanV2RemoteSources;
+ shownBoxes?: ReadonlyMap<string, ShownBox>;
}): Promise<ExtractedFrames[]> {
const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);
const remoteSources = input.remoteSources ?? {};
@@ -192,24 +222,7 @@ export async function extractDeferredVideoFramesForChunk(input: {
});
}
if (deferred.length === 0) return [];
- const table = createFrameLookupTable(
- input.planVideos.videos,
- deferred.map((video) => ({
- ...video,
- outputDir: input.outputRoot,
- framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
- })),
- resolveRenderFpsConfig(input.fps).value,
- );
- const shown = new Map<string, Set<number>>();
- for (const frame of input.frames) {
- const globalTime = (frame * input.fps.den) / input.fps.num;
- for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
- const indexes = shown.get(videoId) ?? new Set<number>();
- indexes.add(payload.frameIndex);
- shown.set(videoId, indexes);
- }
- }
+ const shown = shownDeferredFrameIndexes(input.planVideos, input.frames, input.fps);
return Promise.all(
deferred.map(async (video) => {
const outputDir = join(input.outputRoot, video.videoId);
@@ -236,6 +249,7 @@ export async function extractDeferredVideoFramesForChunk(input: {
outputDir: input.outputRoot,
format: range.format,
frameRanges: contiguousRuns(indexes),
+ shownBox: input.shownBoxes?.get(video.videoId),
},
undefined,
input.cfg,
diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.ts
index 3055e2cdd2..79e6dee3f6 100644
--- a/packages/producer/src/services/distributed/renderChunk.test.ts
+++ b/packages/producer/src/services/distributed/renderChunk.test.ts
@@ -680,6 +680,58 @@ describe("renderChunk() — a video the chunk does not show", () => {
}, 120_000);
});
+describe("renderChunk() — a video shown smaller than its source", () => {
+ it("measures the box each video is drawn in before extracting its frames", async () => {
+ if (!hasChrome) {
+ console.warn(
+ "[renderChunk.test] skipping shown-size test — chrome-headless-shell not available",
+ );
+ return;
+ }
+ const projectDir = join(runRoot, "project-shown-size");
+ mkdirSync(projectDir, { recursive: true });
+ const synth = spawnSync("ffmpeg", [
+ ...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=128x72:r=30:d=2"],
+ ...["-c:v", "libx264", "-pix_fmt", "yuv420p", join(projectDir, "clip.mp4")],
+ ]);
+ if (synth.status !== 0) throw new Error(synth.stderr.toString());
+ writeFileSync(
+ join(projectDir, "index.html"),
+ // Clips over a second long, so the planner defers them to the chunks.
+ FIXTURE_HTML.replace('data-duration="0.16667"', 'data-duration="1.2"').replace(
+ "</div>",
+ '<video id="pip" src="clip.mp4" muted data-start="0" data-duration="1.2" style="position:absolute;right:0;bottom:0;width:32px;height:18px"></video>' +
+ '<video id="full" src="clip.mp4" muted data-start="0" data-duration="1.2" style="position:absolute;left:0;top:0;width:128px;height:72px"></video></div>',
+ ),
+ "utf-8",
+ );
+ const planDir = join(runRoot, "plan-shown-size");
+ await planV2(
+ projectDir,
+ { fps: 30, width: 160, height: 120, format: "png-sequence", chunkSize: 36 },
+ planDir,
+ );
+ const chunkDir = join(runRoot, "chunk0-shown-size");
+ materializePlanV2Target(planDir, { role: "chunk", chunkIndex: 0 }, chunkDir);
+ let result;
+ try {
+ result = await renderChunk(chunkDir, 0, join(runRoot, "chunk0-shown-size-out"));
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err);
+ if (!HOST_CHROME_FAILURE_PATTERNS.test(message)) throw err;
+ console.warn(
+ "[renderChunk.test] skipping shown-size test — host Chrome stack can't render:",
+ message.slice(0, 240),
+ );
+ return;
+ }
+ expect(JSON.parse(readFileSync(result.perfPath, "utf-8")).videoShownBoxes).toEqual({
+ pip: { width: 32, height: 18 },
+ full: { width: 128, height: 72 },
+ });
+ }, 120_000);
+});
+
describe("renderChunk() — frameRange", () => {
// A box sliding across the frame, so every frame differs and a misplaced,
// dropped or repeated frame changes the assembled output.
@@ -1030,6 +1082,30 @@ describe("extractDeferredVideoFramesForChunk", () => {
}
});
+ it("extracts a video's frames at the size it is shown in when given its box", async () => {
+ const [chunk] = await extractDeferredVideoFramesForChunk({
+ planDir,
+ planVideos: planVideos(false),
+ frames: frameSpan(30, 32),
+ fps: { num: 30, den: 1 },
+ outputRoot: join(root, "chunk-shown"),
+ cfg: {},
+ shownBoxes: new Map([["clip", { width: 16, height: 9 }]]),
+ });
+ const [frame] = chunk!.framePaths.values();
+ const size = spawnSync("ffprobe", [
+ "-v",
+ "error",
+ "-show_entries",
+ "stream=width,height",
+ "-of",
+ "csv=p=0",
+ "--",
+ frame!,
+ ]);
+ expect(size.stdout.toString().trim()).toBe("16,10");
+ });
+
// Lossless keeps the file large next to what one seek reads.
const big = join(root, "big.mp4");
const synthBig = () => {
diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts
index aeae789a5c..9d3d23269f 100644
--- a/packages/producer/src/services/distributed/renderChunk.ts
+++ b/packages/producer/src/services/distributed/renderChunk.ts
@@ -53,9 +53,11 @@ import {
type EngineConfig,
getEncoderPreset,
initializeSession,
+ measureVideoShownBoxes,
probeBeginFrameLiveness,
readWebGlVendorInfoFromCanvas,
resolveConfig,
+ type ShownBox,
} from "@hyperframes/engine";
import { defaultLogger } from "../../logger.js";
import { applyRenderWarningPolicy, collectVideoMetadataHints } from "../renderOrchestrator.js";
@@ -100,6 +102,7 @@ import {
MISSING_RUNTIME_ENV_SNAPSHOT,
rebuildExtractedFramesFromPlanDir,
RenderChunkValidationError,
+ shownDeferredFrameIndexes,
} from "./chunkFrames.js";
export {
@@ -623,21 +626,12 @@ export async function renderChunk(
// Beside workDir, not in it: a screenshot retry wipes workDir.
const videoFramesDir = `${workDir}.video-frames`;
- // Build the immutable frame lookup once. Each browser session/worker gets
- // its own injector hook because the hook remembers the last injected frame
- // per video; sharing that state across a fresh retry page can suppress the
- // first injection and produce a blank/stale frame.
- const deferredVideoFrames = planVideos
- ? await extractDeferredVideoFramesForChunk({
- planDir,
- planVideos,
- frames: Array.from({ length: framesInChunk }, (_, i) => range.startFrame + i),
- fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
- outputRoot: videoFramesDir,
- cfg,
- remoteSources,
- })
- : [];
+ const chunkFrameNumbers = Array.from({ length: framesInChunk }, (_, i) => range.startFrame + i);
+ const chunkFps = { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen };
+ const deferredVideos = planVideos?.extracted.filter((video) => video.deferredRange) ?? [];
+ const shownFrames = planVideos
+ ? shownDeferredFrameIndexes(planVideos, chunkFrameNumbers, chunkFps)
+ : new Map<string, Set<number>>();
// The browser cannot load a source read by URL; size its <video> from the plan instead.
const remoteVideoIds = new Set(
planVideos?.extracted
@@ -647,23 +641,6 @@ export async function renderChunk(
)
.map((video) => video.videoId),
);
- const remoteVideos = deferredVideoFrames.filter((video) => remoteVideoIds.has(video.videoId));
- const videoFrameLookup =
- planVideos && planVideos.extracted.length > 0
- ? createFrameLookupTable(
- planVideos.videos,
- [
- ...rebuildExtractedFramesFromPlanDir(
- planDir,
- planVideos.extracted.filter((video) => !video.deferredRange),
- v2Manifest === null ? "dense-v1" : "sparse-v2",
- ),
- ...deferredVideoFrames,
- ],
- resolveRenderFpsConfig(job.config.fps).value,
- )
- : null;
- const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);
// ── File server with the seeded-random shim ──
// `Math.random` / `crypto.getRandomValues` are seeded from virtual
@@ -684,10 +661,14 @@ export async function renderChunk(
videoCount: planVideos?.videos.length ?? 0,
html: readFileSync(join(compiledDir, "index.html"), "utf-8"),
// The plan ships a deferred source only to chunks that show it; don't wait on the rest.
- skipReadinessVideoIds: deferredVideoFrames
- .filter((video) => video.framePaths.size === 0 || remoteVideoIds.has(video.videoId))
+ skipReadinessVideoIds: deferredVideos
+ .filter(
+ (video) => !shownFrames.get(video.videoId)?.size || remoteVideoIds.has(video.videoId),
+ )
.map((video) => video.videoId),
- videoMetadataHints: collectVideoMetadataHints(remoteVideos),
+ videoMetadataHints: collectVideoMetadataHints(
+ deferredVideos.filter((video) => remoteVideoIds.has(video.videoId)),
+ ),
});
// Resolve worker count up-front. Sequential capture reuses the initialized
@@ -717,6 +698,7 @@ export async function renderChunk(
const capturePerfs: CapturePerfSummary[] = [];
const captureAttempts: Parameters<typeof runCaptureStage>[0]["captureAttempts"] = [];
let forceScreenshotForChunk = encoder.forceScreenshot;
+ let shownBoxes: Map<string, ShownBox> | undefined;
try {
if (chunkWorkerCount === 1 || !forceScreenshotForChunk) {
// Sequential capture reuses this session. Parallel BeginFrame capture
@@ -738,6 +720,12 @@ export async function renderChunk(
cfg,
);
sessionBootMs = Date.now() - bootStarted;
+ // Measured before extraction so each video is extracted at the size it is drawn.
+ shownBoxes = await measureVideoShownBoxes(
+ session.page,
+ [...shownFrames.keys()],
+ chunkFrameNumbers.map((frame) => (frame * chunkFps.den) / chunkFps.num),
+ );
// `discardWarmupCapture` is intentionally NOT called: every frame
// seeks fresh DOM, so `lastFrameCache` is never read; priming it
// would deadlock Chrome's compositor by issuing a second beginFrame
@@ -784,6 +772,39 @@ export async function renderChunk(
}
}
+ // Build the immutable frame lookup once. Each browser session/worker gets
+ // its own injector hook because the hook remembers the last injected frame
+ // per video; sharing that state across a fresh retry page can suppress the
+ // first injection and produce a blank/stale frame.
+ const deferredVideoFrames = planVideos
+ ? await extractDeferredVideoFramesForChunk({
+ planDir,
+ planVideos,
+ frames: chunkFrameNumbers,
+ fps: chunkFps,
+ outputRoot: videoFramesDir,
+ cfg,
+ remoteSources,
+ shownBoxes,
+ })
+ : [];
+ const videoFrameLookup =
+ planVideos && planVideos.extracted.length > 0
+ ? createFrameLookupTable(
+ planVideos.videos,
+ [
+ ...rebuildExtractedFramesFromPlanDir(
+ planDir,
+ planVideos.extracted.filter((video) => !video.deferredRange),
+ v2Manifest === null ? "dense-v1" : "sparse-v2",
+ ),
+ ...deferredVideoFrames,
+ ],
+ resolveRenderFpsConfig(job.config.fps).value,
+ )
+ : null;
+ const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);
+
const captureStarted = Date.now();
captureMode = await runCaptureWithScreenshotFallback({
forceScreenshot: forceScreenshotForChunk,
@@ -1009,6 +1030,8 @@ export async function renderChunk(
encodeStageMs,
workers: chunkWorkerCount,
captureMode,
+ // Device-pixel box each video's frames were extracted to cover; a video not listed kept full size.
+ videoShownBoxes: Object.fromEntries(shownBoxes ?? []),
sha256,
outputKind,
producerVersion: plan.producerVersion,