Skip to content

Write HTML. Render video. Built for agents.

perf(engine): re-renders reuse the extracted frames of remote videos (#5408)

Miguel Ángelb6467d91315e
diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc
index 5f0e5f5984..07d33873a9 100644
--- a/.fallowrc.jsonc
+++ b/.fallowrc.jsonc
@@ -466,6 +466,9 @@
// require intrusive middleware changes beyond this PR's scope.
"minLines": 6,
"ignore": [
+ // Pre-existing clone with extractVideosStage.ffmpegTimeout.test.ts: both mock the engine's whole
+ // ExtractionResult. Touching the mock to record another option re-flags it.
+ "packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts",
// AWS Lambda and GCP Cloud Run deliberately mirror the same distributed
// rendering lifecycle while retaining provider-specific SDK, storage, and
// retry semantics. The Plan v2 AWS adapter extends that existing symmetry;
diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.ts
index 01c1840921..58339b923c 100644
--- a/packages/engine/src/services/extractionCache.test.ts
+++ b/packages/engine/src/services/extractionCache.test.ts
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { createHash } from "node:crypto";
import {
existsSync,
mkdirSync,
@@ -24,6 +25,7 @@ import {
markCacheEntryComplete,
partialCacheEntryDir,
publishCacheEntry,
+ readContentSha256,
readKeyStat,
rehydrateCacheEntry,
touchCacheDir,
@@ -139,6 +141,34 @@ describe("computeCacheKey", () => {
expect(key).toMatch(/^[0-9a-f]{64}$/);
});
+ it("keys a path-keyed source exactly as before content keys existed", () => {
+ const input = base(sourceFile);
+ const legacyBlob = JSON.stringify({
+ p: input.videoPath,
+ m: input.mtimeMs,
+ s: input.size,
+ ms: 0,
+ d: 3,
+ f: "30",
+ fmt: "jpg",
+ });
+ expect(computeCacheKey(input)).toBe(createHash("sha256").update(legacyBlob).digest("hex"));
+ });
+
+ it("keys a content-addressed source by its bytes, not its path or mtime", () => {
+ const moved = join(tmpRoot, "moved.mp4");
+ writeFileSync(moved, "fake-video-bytes", "utf-8");
+ const original = computeCacheKey({ ...base(sourceFile), contentSha256: "a1" });
+ expect(computeCacheKey({ ...base(moved), mtimeMs: 1, contentSha256: "a1" })).toBe(original);
+ expect(computeCacheKey({ ...base(sourceFile), contentSha256: "b2" })).not.toBe(original);
+ });
+
+ it("hashes a file's bytes for its content key, or skips the cache when unreadable", async () => {
+ const expected = createHash("sha256").update("fake-video-bytes").digest("hex");
+ expect(await readContentSha256(sourceFile)).toBe(expected);
+ expect(await readContentSha256(join(tmpRoot, "missing.mp4"))).toBeNull();
+ });
+
it("changes when path changes (moved files re-extract)", () => {
const other = join(tmpRoot, "other.mp4");
writeFileSync(other, "fake-video-bytes", "utf-8");
diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.ts
index f7a12406e2..ab2297404f 100644
--- a/packages/engine/src/services/extractionCache.ts
+++ b/packages/engine/src/services/extractionCache.ts
@@ -34,6 +34,7 @@
import { createHash, randomUUID } from "node:crypto";
import {
+ createReadStream,
existsSync,
lstatSync,
mkdirSync,
@@ -88,6 +89,8 @@ export interface CacheKeyInput {
format: CacheFrameFormat;
/** Optional source transform applied during extraction. */
transform?: string;
+ /** SHA-256 of the source bytes, in place of path and mtime (downloaded media gets new ones each render). */
+ contentSha256?: string;
}
export interface CacheEntry {
@@ -110,6 +113,19 @@ export interface CachePublishResult {
published: boolean;
}
+export async function readContentSha256(
+ path: string,
+ signal?: AbortSignal,
+): Promise<string | null> {
+ try {
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(path, { signal })) hash.update(chunk);
+ return hash.digest("hex");
+ } catch {
+ return null;
+ }
+}
+
/**
* Read `(mtimeMs, size)` for a path. Returns `null` if the file is missing —
* callers should skip the cache path for that entry so the extractor surfaces
@@ -128,18 +144,13 @@ export function readKeyStat(videoPath: string): { mtimeMs: number; size: number
function canonicalKeyBlob(input: CacheKeyInput): string {
const durationForKey = Number.isFinite(input.duration) ? input.duration : -1;
- const blob: {
- p: string;
- m: number;
- s: number;
- ms: number;
- d: number;
- f: string;
- fmt: CacheFrameFormat;
- t?: string;
- } = {
- p: input.videoPath,
- m: input.mtimeMs,
+ // Identity first, so keys of path-keyed entries stay byte-identical.
+ const identity: Record<string, string | number> =
+ input.contentSha256 !== undefined
+ ? { c: input.contentSha256 }
+ : { p: input.videoPath, m: input.mtimeMs };
+ const blob: Record<string, string | number> = {
+ ...identity,
s: input.size,
ms: input.mediaStart,
d: durationForKey,
diff --git a/packages/engine/src/services/videoFrameExtractor.downloadCache.test.ts b/packages/engine/src/services/videoFrameExtractor.downloadCache.test.ts
new file mode 100644
index 0000000000..94bf74486b
--- /dev/null
+++ b/packages/engine/src/services/videoFrameExtractor.downloadCache.test.ts
@@ -0,0 +1,181 @@
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import type { VideoElement } from "../types.js";
+import { runFfmpeg } from "../utils/runFfmpeg.js";
+
+// Each render gets its own copy of the clip: new path and mtime, same bytes.
+const source = vi.hoisted(() => ({ clip: "" }));
+vi.mock("../utils/urlDownloader.js", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../utils/urlDownloader.js")>()),
+ downloadToTemp: async (_url: string, destDir: string) => {
+ const local = join(destDir, "clip.mp4");
+ copyFileSync(source.clip, local);
+ return local;
+ },
+}));
+
+const hashing = vi.hoisted(() => ({ paths: [] as string[] }));
+vi.mock("./extractionCache.js", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("./extractionCache.js")>();
+ return {
+ ...actual,
+ readContentSha256: async (path: string) => {
+ hashing.paths.push(path);
+ return actual.readContentSha256(path);
+ },
+ };
+});
+
+const { extractAllVideoFrames } = await import("./videoFrameExtractor.js");
+
+describe.skipIf(process.platform === "win32")("extractAllVideoFrames downloaded sources", () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-extract-render-owned-"));
+ const cacheDir = join(dir, "cache");
+ let red = "";
+ let blue = "";
+ let green = "";
+
+ async function synth(color: string, seconds = 1): Promise<string> {
+ const clip = join(dir, `${color}-${seconds}s.mp4`);
+ const made = await runFfmpeg([
+ ...["-y", "-v", "error", "-f", "lavfi", "-i", `color=c=${color}:s=32x16:d=${seconds}:r=30`],
+ ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", clip],
+ ]);
+ expect(made.success, made.stderr).toBe(true);
+ return clip;
+ }
+
+ function clipElement(src: string, id = "clip", end = 1): VideoElement {
+ return { id, src, start: 0, end, mediaStart: 0, loop: false, hasAudio: false };
+ }
+
+ // The producer downloads a remote src at compile time into the render's compiled dir;
+ // a distributed plan also copies the whole project there (`subdir: "."`).
+ function renderCompiledCopy(
+ name: string,
+ videos = [clipElement("_remote_media/clip.mp4")],
+ {
+ deferRangeExtraction = false,
+ subdir = "_remote_media",
+ config = { extractCacheDir: cacheDir } as { extractCacheDir?: string },
+ } = {},
+ ) {
+ const compiledDir = join(dir, name, "compiled");
+ mkdirSync(join(compiledDir, subdir), { recursive: true });
+ copyFileSync(source.clip, join(compiledDir, subdir, "clip.mp4"));
+ const outputDir = join(compiledDir, "frames");
+ return extractAllVideoFrames(
+ videos,
+ join(dir, "project"),
+ {
+ ...{ fps: 30, outputDir, deferRangeExtraction },
+ contentKeyedDirs: [join(compiledDir, "_remote_media")],
+ },
+ undefined,
+ config,
+ compiledDir,
+ );
+ }
+
+ function firstFrameBytes(result: Awaited<ReturnType<typeof extractAllVideoFrames>>): Buffer {
+ const frame = result.extracted[0]?.framePaths.get(0);
+ if (!frame) throw new Error("missing first frame");
+ return readFileSync(frame);
+ }
+
+ function renderRemoteSrc(name: string) {
+ const outputDir = join(dir, name);
+ mkdirSync(outputDir, { recursive: true });
+ return extractAllVideoFrames(
+ [clipElement("https://cdn.example.com/clip.mp4")],
+ dir,
+ { fps: 30, outputDir },
+ undefined,
+ { extractCacheDir: cacheDir },
+ );
+ }
+
+ beforeAll(async () => {
+ mkdirSync(join(dir, "project"), { recursive: true });
+ red = await synth("red");
+ blue = await synth("blue");
+ green = await synth("green");
+ });
+
+ afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+ it("a re-render reuses the frames of a video copied into its compiled dir", async () => {
+ source.clip = red;
+ const first = await renderCompiledCopy("compiled-1");
+ const second = await renderCompiledCopy("compiled-2");
+ expect(first.errors).toEqual([]);
+ expect(first.phaseBreakdown.cacheMisses).toBe(1);
+ expect(second.errors).toEqual([]);
+ expect(second.phaseBreakdown.cacheHits).toBe(1);
+ expect(second.phaseBreakdown.cacheMisses).toBe(0);
+ expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames);
+ expect(firstFrameBytes(second).equals(firstFrameBytes(first))).toBe(true);
+ });
+
+ it("a re-render reuses the frames of a video the extractor downloaded itself", async () => {
+ source.clip = blue;
+ await renderRemoteSrc("remote-1");
+ const second = await renderRemoteSrc("remote-2");
+ expect(second.errors).toEqual([]);
+ expect(second.phaseBreakdown.cacheHits).toBe(1);
+ });
+
+ it("a render-owned video whose bytes changed extracts again", async () => {
+ source.clip = red;
+ await renderCompiledCopy("changed-1");
+ source.clip = green;
+ const changed = await renderCompiledCopy("changed-2");
+ expect(changed.errors).toEqual([]);
+ expect(changed.phaseBreakdown.cacheHits).toBe(0);
+ expect(changed.phaseBreakdown.cacheMisses).toBe(1);
+ });
+
+ it("hashes a file once however many videos use it", async () => {
+ source.clip = red;
+ hashing.paths = [];
+ const result = await renderCompiledCopy("shared", [
+ clipElement("_remote_media/clip.mp4", "a"),
+ { ...clipElement("_remote_media/clip.mp4", "b"), start: 1, mediaStart: 0.5, end: 1.5 },
+ ]);
+ expect(result.errors).toEqual([]);
+ expect(hashing.paths).toHaveLength(1);
+ });
+
+ it("does not hash a video whose extraction is deferred", async () => {
+ source.clip = await synth("yellow", 3);
+ hashing.paths = [];
+ const result = await renderCompiledCopy(
+ "deferred",
+ [clipElement("_remote_media/clip.mp4", "clip", 3)],
+ { deferRangeExtraction: true },
+ );
+ expect(result.errors).toEqual([]);
+ expect(result.extracted[0]?.deferredRange).toBeDefined();
+ expect(hashing.paths).toEqual([]);
+ });
+
+ it("keeps a project copy outside the download dirs on its path key", async () => {
+ source.clip = red;
+ hashing.paths = [];
+ const first = await renderCompiledCopy("copy-1", [clipElement("clip.mp4")], { subdir: "." });
+ const second = await renderCompiledCopy("copy-2", [clipElement("clip.mp4")], { subdir: "." });
+ expect(second.errors).toEqual([]);
+ expect(first.phaseBreakdown.cacheHits + second.phaseBreakdown.cacheHits).toBe(0);
+ expect(hashing.paths).toEqual([]);
+ });
+
+ it("does not hash when the cache is off", async () => {
+ source.clip = red;
+ hashing.paths = [];
+ const result = await renderCompiledCopy("cache-off", undefined, { config: {} });
+ expect(result.errors).toEqual([]);
+ expect(hashing.paths).toEqual([]);
+ });
+});
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index bdf26ac4e0..76e9ea43cd 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -12,6 +12,7 @@ import { join } from "path";
import { cpus } from "os";
import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";
import { parseHTML } from "linkedom";
+import { isPathInside } from "@hyperframes/parsers/asset-paths";
import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";
import {
MEDIA_RENDER_ID_ATTR,
@@ -67,6 +68,7 @@ import {
lookupCacheEntry,
partialCacheEntryDir,
publishCacheEntry,
+ readContentSha256,
readKeyStat,
rehydrateCacheEntry,
touchCacheEntry,
@@ -243,8 +245,12 @@ export interface ExtractionOptions {
* `totalFrames` so another process can extract just the frames it needs with `frameRanges`.
*/
deferRangeExtraction?: boolean;
+ /** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
+ contentKeyedDirs?: readonly string[];
}
+const DOWNLOAD_SUBDIR = "_downloads";
+
export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
@@ -1870,7 +1876,7 @@ export async function extractAllVideoFrames(
}
if (isHttpUrl(videoPath)) {
- const downloadDir = join(options.outputDir, "_downloads");
+ const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);
mkdirSync(downloadDir, { recursive: true });
videoPath = await downloadToTemp(
videoPath,
@@ -1925,6 +1931,11 @@ export async function extractAllVideoFrames(
// HDR preflight. Without this, every render would write a new
// normalized file with a fresh mtime → fresh cache key → perpetual misses.
// Phase 3 updates mediaStart after trimming any invisible negative preroll.
+ // Downloads land at a new path and mtime every render, so they are keyed by content instead.
+ const contentKeyedDirs = [
+ join(options.outputDir, DOWNLOAD_SUBDIR),
+ ...(options.contentKeyedDirs ?? []),
+ ];
const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => {
const stat = readKeyStat(videoPath);
// Missing files return null — skip the cache path for that entry. The
@@ -1937,6 +1948,7 @@ export async function extractAllVideoFrames(
mtimeMs: stat.mtimeMs,
size: stat.size,
mediaStart: video.mediaStart,
+ contentKeyed: contentKeyedDirs.some((dir) => isPathInside(videoPath, dir)),
};
});
@@ -2140,10 +2152,30 @@ export async function extractAllVideoFrames(
return { ...rehydrated, ownedByLookup: true };
}
+ // Hashed only for works that reach a lookup (deferred ranges never do), once per file.
+ const contentSha256ByPath = new Map<string, string | null>();
+ async function hashContentKeyedSources(works: PreparedExtraction[]): Promise<void> {
+ if (!cacheRootDir) return;
+ const paths = new Set<string>();
+ for (const work of works) {
+ const keyInput = cacheKeyInputs[work.index];
+ if (keyInput?.contentKeyed) paths.add(keyInput.videoPath);
+ }
+ await Promise.all(
+ [...paths].map(async (path) =>
+ contentSha256ByPath.set(path, await readContentSha256(path, signal)),
+ ),
+ );
+ }
+
function lookupCacheFor(work: PreparedExtraction): ExtractionOutcome | UniqueExtractionMiss {
if (!cacheRootDir) return { work };
const keyInput = cacheKeyInputs[work.index];
if (!keyInput) return { work };
+ const contentSha256 = keyInput.contentKeyed
+ ? (contentSha256ByPath.get(keyInput.videoPath) ?? null)
+ : undefined;
+ if (contentSha256 === null) return { work };
const transformParts = [
work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
work.hdrToSdrTransformKey,
@@ -2156,6 +2188,7 @@ export async function extractAllVideoFrames(
mtimeMs: keyInput.mtimeMs,
size: keyInput.size,
mediaStart: keyInput.mediaStart,
+ contentSha256,
duration: work.videoDuration,
fps: fpsKey,
format: work.format,
@@ -2473,8 +2506,11 @@ export async function extractAllVideoFrames(
});
}
const cacheMisses: UniqueExtractionMiss[] = [];
- for (const work of uniqueWorks.values()) {
- if (uniqueOutcomes.has(work.dedupeKey)) continue;
+ const worksToLookUp = [...uniqueWorks.values()].filter(
+ (work) => !uniqueOutcomes.has(work.dedupeKey),
+ );
+ await hashContentKeyedSources(worksToLookUp);
+ for (const work of worksToLookUp) {
const lookup = lookupCacheFor(work);
if ("work" in lookup) {
cacheMisses.push(lookup);
diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts
index 4c8417c954..f898df0b63 100644
--- a/packages/producer/src/services/htmlCompiler.ts
+++ b/packages/producer/src/services/htmlCompiler.ts
@@ -1361,7 +1361,7 @@ export function collectExternalAssets(
};
}
-const REMOTE_MEDIA_SUBDIR = "_remote_media";
+export const REMOTE_MEDIA_SUBDIR = "_remote_media";
// Match opening tags of <video> or <audio> elements that carry an HTTP(S) src.
// Uses [^>]* to span attributes — safe for composition elements that won't
// have `>` inside quoted attribute values (data-title etc.).
diff --git a/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts
index b4e9a15580..d16a2ef5e7 100644
--- a/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts
+++ b/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts
@@ -11,6 +11,7 @@ const extractionCalls = vi.hoisted(
() => new Array<{ timelineEnd: number | undefined; durationSeconds: number }>(),
);
const toneMapHdrToSdrCalls = vi.hoisted(() => new Array<boolean | undefined>());
+const contentKeyedDirsCalls = vi.hoisted(() => new Array<readonly string[] | undefined>());
const fixtureState = vi.hoisted(() => ({ sourceDurationSeconds: 60 }));
vi.mock("@hyperframes/engine", async (importOriginal) => {
@@ -20,7 +21,11 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
extractAllVideoFrames: async (
videos: VideoElement[],
_baseDir: string,
- options: { timelineEnd?: number; toneMapHdrToSdr?: boolean },
+ options: {
+ timelineEnd?: number;
+ toneMapHdrToSdr?: boolean;
+ contentKeyedDirs?: readonly string[];
+ },
): Promise<ExtractionResult> => {
const sourceDurationSeconds = fixtureState.sourceDurationSeconds;
const video = videos[0];
@@ -38,6 +43,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
video.end = video.start + durationSeconds;
extractionCalls.push({ timelineEnd: options.timelineEnd, durationSeconds });
toneMapHdrToSdrCalls.push(options.toneMapHdrToSdr);
+ contentKeyedDirsCalls.push(options.contentKeyedDirs);
return {
success: true,
extracted: [],
@@ -66,6 +72,8 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
};
});
+import { join } from "node:path";
+import { REMOTE_MEDIA_SUBDIR } from "../../htmlCompiler.js";
import { createRenderJob } from "../../renderOrchestrator.js";
import { runExtractVideosStage } from "./extractVideosStage.js";
@@ -141,6 +149,16 @@ describe.each([
expect(toneMapHdrToSdrCalls).toEqual([true]);
});
+
+ it("caches the compiled remote-media copies by content", async () => {
+ contentKeyedDirsCalls.splice(0);
+
+ await runStage(2, materializeSymlinks);
+
+ expect(contentKeyedDirsCalls).toEqual([
+ [join("/tmp/hf-timeline-bound-compiled", REMOTE_MEDIA_SUBDIR)],
+ ]);
+ });
});
describe("video extraction source logging", () => {
diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.ts
index 994249f572..877b602cfd 100644
--- a/packages/producer/src/services/render/stages/extractVideosStage.ts
+++ b/packages/producer/src/services/render/stages/extractVideosStage.ts
@@ -56,6 +56,7 @@ import {
type RenderJob,
} from "../../renderOrchestrator.js";
import { materializeExtractedFramesForCompiledDir, type CompositionMetadata } from "../shared.js";
+import { REMOTE_MEDIA_SUBDIR } from "../../htmlCompiler.js";
import { resolveRenderFpsConfig } from "../../fileServer.js";
import type { ProducerLogger } from "../../../logger.js";
import { encoderFailureError } from "../encoderInterruption.js";
@@ -504,6 +505,7 @@ export async function runExtractVideosStage(
maxTransientRetries: extractionPolicy.maxTransientRetries,
collectProbeFailures: extractionPolicy.failureMode === "enforce",
deferRangeExtraction,
+ contentKeyedDirs: [join(compiledDir, REMOTE_MEDIA_SUBDIR)],
},
abortSignal,
cfg,