Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(engine): re-renders reuse the extracted frames of remote videos (#5408)
Miguel Ángelb6467d91315e
diff --git a/.fallowrc.jsonc b/.fallowrc.jsoncindex 5f0e5f5984..07d33873a9 100644--- a/.fallowrc.jsonc+++ b/.fallowrc.jsonc@@ -466,6 +466,9 @@// require intrusive middleware changes beyond this PR's scope."minLines": 6,"ignore": [+ // Pre-existing clone with extractVideosStage.ffmpegTimeout.test.ts: both mock the engine's whole+ // ExtractionResult. Touching the mock to record another option re-flags it.+ "packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts",// AWS Lambda and GCP Cloud Run deliberately mirror the same distributed// rendering lifecycle while retaining provider-specific SDK, storage, and// retry semantics. The Plan v2 AWS adapter extends that existing symmetry;diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.tsindex 01c1840921..58339b923c 100644--- a/packages/engine/src/services/extractionCache.test.ts+++ b/packages/engine/src/services/extractionCache.test.ts@@ -1,4 +1,5 @@import { afterEach, beforeEach, describe, expect, it } from "vitest";+import { createHash } from "node:crypto";import {existsSync,mkdirSync,@@ -24,6 +25,7 @@ import {markCacheEntryComplete,partialCacheEntryDir,publishCacheEntry,+ readContentSha256,readKeyStat,rehydrateCacheEntry,touchCacheDir,@@ -139,6 +141,34 @@ describe("computeCacheKey", () => {expect(key).toMatch(/^[0-9a-f]{64}$/);});+ it("keys a path-keyed source exactly as before content keys existed", () => {+ const input = base(sourceFile);+ const legacyBlob = JSON.stringify({+ p: input.videoPath,+ m: input.mtimeMs,+ s: input.size,+ ms: 0,+ d: 3,+ f: "30",+ fmt: "jpg",+ });+ expect(computeCacheKey(input)).toBe(createHash("sha256").update(legacyBlob).digest("hex"));+ });++ it("keys a content-addressed source by its bytes, not its path or mtime", () => {+ const moved = join(tmpRoot, "moved.mp4");+ writeFileSync(moved, "fake-video-bytes", "utf-8");+ const original = computeCacheKey({ ...base(sourceFile), contentSha256: "a1" });+ expect(computeCacheKey({ ...base(moved), mtimeMs: 1, contentSha256: "a1" })).toBe(original);+ expect(computeCacheKey({ ...base(sourceFile), contentSha256: "b2" })).not.toBe(original);+ });++ it("hashes a file's bytes for its content key, or skips the cache when unreadable", async () => {+ const expected = createHash("sha256").update("fake-video-bytes").digest("hex");+ expect(await readContentSha256(sourceFile)).toBe(expected);+ expect(await readContentSha256(join(tmpRoot, "missing.mp4"))).toBeNull();+ });+it("changes when path changes (moved files re-extract)", () => {const other = join(tmpRoot, "other.mp4");writeFileSync(other, "fake-video-bytes", "utf-8");diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.tsindex f7a12406e2..ab2297404f 100644--- a/packages/engine/src/services/extractionCache.ts+++ b/packages/engine/src/services/extractionCache.ts@@ -34,6 +34,7 @@import { createHash, randomUUID } from "node:crypto";import {+ createReadStream,existsSync,lstatSync,mkdirSync,@@ -88,6 +89,8 @@ export interface CacheKeyInput {format: CacheFrameFormat;/** Optional source transform applied during extraction. */transform?: string;+ /** SHA-256 of the source bytes, in place of path and mtime (downloaded media gets new ones each render). */+ contentSha256?: string;}export interface CacheEntry {@@ -110,6 +113,19 @@ export interface CachePublishResult {published: boolean;}+export async function readContentSha256(+ path: string,+ signal?: AbortSignal,+): Promise<string | null> {+ try {+ const hash = createHash("sha256");+ for await (const chunk of createReadStream(path, { signal })) hash.update(chunk);+ return hash.digest("hex");+ } catch {+ return null;+ }+}+/*** Read `(mtimeMs, size)` for a path. Returns `null` if the file is missing —* callers should skip the cache path for that entry so the extractor surfaces@@ -128,18 +144,13 @@ export function readKeyStat(videoPath: string): { mtimeMs: number; size: numberfunction canonicalKeyBlob(input: CacheKeyInput): string {const durationForKey = Number.isFinite(input.duration) ? input.duration : -1;- const blob: {- p: string;- m: number;- s: number;- ms: number;- d: number;- f: string;- fmt: CacheFrameFormat;- t?: string;- } = {- p: input.videoPath,- m: input.mtimeMs,+ // Identity first, so keys of path-keyed entries stay byte-identical.+ const identity: Record<string, string | number> =+ input.contentSha256 !== undefined+ ? { c: input.contentSha256 }+ : { p: input.videoPath, m: input.mtimeMs };+ const blob: Record<string, string | number> = {+ ...identity,s: input.size,ms: input.mediaStart,d: durationForKey,diff --git a/packages/engine/src/services/videoFrameExtractor.downloadCache.test.ts b/packages/engine/src/services/videoFrameExtractor.downloadCache.test.tsnew file mode 100644index 0000000000..94bf74486b--- /dev/null+++ b/packages/engine/src/services/videoFrameExtractor.downloadCache.test.ts@@ -0,0 +1,181 @@+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";+import type { VideoElement } from "../types.js";+import { runFfmpeg } from "../utils/runFfmpeg.js";++// Each render gets its own copy of the clip: new path and mtime, same bytes.+const source = vi.hoisted(() => ({ clip: "" }));+vi.mock("../utils/urlDownloader.js", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../utils/urlDownloader.js")>()),+ downloadToTemp: async (_url: string, destDir: string) => {+ const local = join(destDir, "clip.mp4");+ copyFileSync(source.clip, local);+ return local;+ },+}));++const hashing = vi.hoisted(() => ({ paths: [] as string[] }));+vi.mock("./extractionCache.js", async (importOriginal) => {+ const actual = await importOriginal<typeof import("./extractionCache.js")>();+ return {+ ...actual,+ readContentSha256: async (path: string) => {+ hashing.paths.push(path);+ return actual.readContentSha256(path);+ },+ };+});++const { extractAllVideoFrames } = await import("./videoFrameExtractor.js");++describe.skipIf(process.platform === "win32")("extractAllVideoFrames downloaded sources", () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-extract-render-owned-"));+ const cacheDir = join(dir, "cache");+ let red = "";+ let blue = "";+ let green = "";++ async function synth(color: string, seconds = 1): Promise<string> {+ const clip = join(dir, `${color}-${seconds}s.mp4`);+ const made = await runFfmpeg([+ ...["-y", "-v", "error", "-f", "lavfi", "-i", `color=c=${color}:s=32x16:d=${seconds}:r=30`],+ ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", clip],+ ]);+ expect(made.success, made.stderr).toBe(true);+ return clip;+ }++ function clipElement(src: string, id = "clip", end = 1): VideoElement {+ return { id, src, start: 0, end, mediaStart: 0, loop: false, hasAudio: false };+ }++ // The producer downloads a remote src at compile time into the render's compiled dir;+ // a distributed plan also copies the whole project there (`subdir: "."`).+ function renderCompiledCopy(+ name: string,+ videos = [clipElement("_remote_media/clip.mp4")],+ {+ deferRangeExtraction = false,+ subdir = "_remote_media",+ config = { extractCacheDir: cacheDir } as { extractCacheDir?: string },+ } = {},+ ) {+ const compiledDir = join(dir, name, "compiled");+ mkdirSync(join(compiledDir, subdir), { recursive: true });+ copyFileSync(source.clip, join(compiledDir, subdir, "clip.mp4"));+ const outputDir = join(compiledDir, "frames");+ return extractAllVideoFrames(+ videos,+ join(dir, "project"),+ {+ ...{ fps: 30, outputDir, deferRangeExtraction },+ contentKeyedDirs: [join(compiledDir, "_remote_media")],+ },+ undefined,+ config,+ compiledDir,+ );+ }++ function firstFrameBytes(result: Awaited<ReturnType<typeof extractAllVideoFrames>>): Buffer {+ const frame = result.extracted[0]?.framePaths.get(0);+ if (!frame) throw new Error("missing first frame");+ return readFileSync(frame);+ }++ function renderRemoteSrc(name: string) {+ const outputDir = join(dir, name);+ mkdirSync(outputDir, { recursive: true });+ return extractAllVideoFrames(+ [clipElement("https://cdn.example.com/clip.mp4")],+ dir,+ { fps: 30, outputDir },+ undefined,+ { extractCacheDir: cacheDir },+ );+ }++ beforeAll(async () => {+ mkdirSync(join(dir, "project"), { recursive: true });+ red = await synth("red");+ blue = await synth("blue");+ green = await synth("green");+ });++ afterAll(() => rmSync(dir, { recursive: true, force: true }));++ it("a re-render reuses the frames of a video copied into its compiled dir", async () => {+ source.clip = red;+ const first = await renderCompiledCopy("compiled-1");+ const second = await renderCompiledCopy("compiled-2");+ expect(first.errors).toEqual([]);+ expect(first.phaseBreakdown.cacheMisses).toBe(1);+ expect(second.errors).toEqual([]);+ expect(second.phaseBreakdown.cacheHits).toBe(1);+ expect(second.phaseBreakdown.cacheMisses).toBe(0);+ expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames);+ expect(firstFrameBytes(second).equals(firstFrameBytes(first))).toBe(true);+ });++ it("a re-render reuses the frames of a video the extractor downloaded itself", async () => {+ source.clip = blue;+ await renderRemoteSrc("remote-1");+ const second = await renderRemoteSrc("remote-2");+ expect(second.errors).toEqual([]);+ expect(second.phaseBreakdown.cacheHits).toBe(1);+ });++ it("a render-owned video whose bytes changed extracts again", async () => {+ source.clip = red;+ await renderCompiledCopy("changed-1");+ source.clip = green;+ const changed = await renderCompiledCopy("changed-2");+ expect(changed.errors).toEqual([]);+ expect(changed.phaseBreakdown.cacheHits).toBe(0);+ expect(changed.phaseBreakdown.cacheMisses).toBe(1);+ });++ it("hashes a file once however many videos use it", async () => {+ source.clip = red;+ hashing.paths = [];+ const result = await renderCompiledCopy("shared", [+ clipElement("_remote_media/clip.mp4", "a"),+ { ...clipElement("_remote_media/clip.mp4", "b"), start: 1, mediaStart: 0.5, end: 1.5 },+ ]);+ expect(result.errors).toEqual([]);+ expect(hashing.paths).toHaveLength(1);+ });++ it("does not hash a video whose extraction is deferred", async () => {+ source.clip = await synth("yellow", 3);+ hashing.paths = [];+ const result = await renderCompiledCopy(+ "deferred",+ [clipElement("_remote_media/clip.mp4", "clip", 3)],+ { deferRangeExtraction: true },+ );+ expect(result.errors).toEqual([]);+ expect(result.extracted[0]?.deferredRange).toBeDefined();+ expect(hashing.paths).toEqual([]);+ });++ it("keeps a project copy outside the download dirs on its path key", async () => {+ source.clip = red;+ hashing.paths = [];+ const first = await renderCompiledCopy("copy-1", [clipElement("clip.mp4")], { subdir: "." });+ const second = await renderCompiledCopy("copy-2", [clipElement("clip.mp4")], { subdir: "." });+ expect(second.errors).toEqual([]);+ expect(first.phaseBreakdown.cacheHits + second.phaseBreakdown.cacheHits).toBe(0);+ expect(hashing.paths).toEqual([]);+ });++ it("does not hash when the cache is off", async () => {+ source.clip = red;+ hashing.paths = [];+ const result = await renderCompiledCopy("cache-off", undefined, { config: {} });+ expect(result.errors).toEqual([]);+ expect(hashing.paths).toEqual([]);+ });+});diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex bdf26ac4e0..76e9ea43cd 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -12,6 +12,7 @@ import { join } from "path";import { cpus } from "os";import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";import { parseHTML } from "linkedom";+import { isPathInside } from "@hyperframes/parsers/asset-paths";import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";import {MEDIA_RENDER_ID_ATTR,@@ -67,6 +68,7 @@ import {lookupCacheEntry,partialCacheEntryDir,publishCacheEntry,+ readContentSha256,readKeyStat,rehydrateCacheEntry,touchCacheEntry,@@ -243,8 +245,12 @@ export interface ExtractionOptions {* `totalFrames` so another process can extract just the frames it needs with `frameRanges`.*/deferRangeExtraction?: boolean;+ /** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */+ contentKeyedDirs?: readonly string[];}+const DOWNLOAD_SUBDIR = "_downloads";+export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;const GC_STALENESS_MS = 24 * 60 * 60 * 1000;const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";@@ -1870,7 +1876,7 @@ export async function extractAllVideoFrames(}if (isHttpUrl(videoPath)) {- const downloadDir = join(options.outputDir, "_downloads");+ const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);mkdirSync(downloadDir, { recursive: true });videoPath = await downloadToTemp(videoPath,@@ -1925,6 +1931,11 @@ export async function extractAllVideoFrames(// HDR preflight. Without this, every render would write a new// normalized file with a fresh mtime → fresh cache key → perpetual misses.// Phase 3 updates mediaStart after trimming any invisible negative preroll.+ // Downloads land at a new path and mtime every render, so they are keyed by content instead.+ const contentKeyedDirs = [+ join(options.outputDir, DOWNLOAD_SUBDIR),+ ...(options.contentKeyedDirs ?? []),+ ];const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => {const stat = readKeyStat(videoPath);// Missing files return null — skip the cache path for that entry. The@@ -1937,6 +1948,7 @@ export async function extractAllVideoFrames(mtimeMs: stat.mtimeMs,size: stat.size,mediaStart: video.mediaStart,+ contentKeyed: contentKeyedDirs.some((dir) => isPathInside(videoPath, dir)),};});@@ -2140,10 +2152,30 @@ export async function extractAllVideoFrames(return { ...rehydrated, ownedByLookup: true };}+ // Hashed only for works that reach a lookup (deferred ranges never do), once per file.+ const contentSha256ByPath = new Map<string, string | null>();+ async function hashContentKeyedSources(works: PreparedExtraction[]): Promise<void> {+ if (!cacheRootDir) return;+ const paths = new Set<string>();+ for (const work of works) {+ const keyInput = cacheKeyInputs[work.index];+ if (keyInput?.contentKeyed) paths.add(keyInput.videoPath);+ }+ await Promise.all(+ [...paths].map(async (path) =>+ contentSha256ByPath.set(path, await readContentSha256(path, signal)),+ ),+ );+ }+function lookupCacheFor(work: PreparedExtraction): ExtractionOutcome | UniqueExtractionMiss {if (!cacheRootDir) return { work };const keyInput = cacheKeyInputs[work.index];if (!keyInput) return { work };+ const contentSha256 = keyInput.contentKeyed+ ? (contentSha256ByPath.get(keyInput.videoPath) ?? null)+ : undefined;+ if (contentSha256 === null) return { work };const transformParts = [work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,work.hdrToSdrTransformKey,@@ -2156,6 +2188,7 @@ export async function extractAllVideoFrames(mtimeMs: keyInput.mtimeMs,size: keyInput.size,mediaStart: keyInput.mediaStart,+ contentSha256,duration: work.videoDuration,fps: fpsKey,format: work.format,@@ -2473,8 +2506,11 @@ export async function extractAllVideoFrames(});}const cacheMisses: UniqueExtractionMiss[] = [];- for (const work of uniqueWorks.values()) {- if (uniqueOutcomes.has(work.dedupeKey)) continue;+ const worksToLookUp = [...uniqueWorks.values()].filter(+ (work) => !uniqueOutcomes.has(work.dedupeKey),+ );+ await hashContentKeyedSources(worksToLookUp);+ for (const work of worksToLookUp) {const lookup = lookupCacheFor(work);if ("work" in lookup) {cacheMisses.push(lookup);diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.tsindex 4c8417c954..f898df0b63 100644--- a/packages/producer/src/services/htmlCompiler.ts+++ b/packages/producer/src/services/htmlCompiler.ts@@ -1361,7 +1361,7 @@ export function collectExternalAssets(};}-const REMOTE_MEDIA_SUBDIR = "_remote_media";+export const REMOTE_MEDIA_SUBDIR = "_remote_media";// Match opening tags of <video> or <audio> elements that carry an HTTP(S) src.// Uses [^>]* to span attributes — safe for composition elements that won't// have `>` inside quoted attribute values (data-title etc.).diff --git a/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.tsindex b4e9a15580..d16a2ef5e7 100644--- a/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts+++ b/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts@@ -11,6 +11,7 @@ const extractionCalls = vi.hoisted(() => new Array<{ timelineEnd: number | undefined; durationSeconds: number }>(),);const toneMapHdrToSdrCalls = vi.hoisted(() => new Array<boolean | undefined>());+const contentKeyedDirsCalls = vi.hoisted(() => new Array<readonly string[] | undefined>());const fixtureState = vi.hoisted(() => ({ sourceDurationSeconds: 60 }));vi.mock("@hyperframes/engine", async (importOriginal) => {@@ -20,7 +21,11 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {extractAllVideoFrames: async (videos: VideoElement[],_baseDir: string,- options: { timelineEnd?: number; toneMapHdrToSdr?: boolean },+ options: {+ timelineEnd?: number;+ toneMapHdrToSdr?: boolean;+ contentKeyedDirs?: readonly string[];+ },): Promise<ExtractionResult> => {const sourceDurationSeconds = fixtureState.sourceDurationSeconds;const video = videos[0];@@ -38,6 +43,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {video.end = video.start + durationSeconds;extractionCalls.push({ timelineEnd: options.timelineEnd, durationSeconds });toneMapHdrToSdrCalls.push(options.toneMapHdrToSdr);+ contentKeyedDirsCalls.push(options.contentKeyedDirs);return {success: true,extracted: [],@@ -66,6 +72,8 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {};});+import { join } from "node:path";+import { REMOTE_MEDIA_SUBDIR } from "../../htmlCompiler.js";import { createRenderJob } from "../../renderOrchestrator.js";import { runExtractVideosStage } from "./extractVideosStage.js";@@ -141,6 +149,16 @@ describe.each([expect(toneMapHdrToSdrCalls).toEqual([true]);});++ it("caches the compiled remote-media copies by content", async () => {+ contentKeyedDirsCalls.splice(0);++ await runStage(2, materializeSymlinks);++ expect(contentKeyedDirsCalls).toEqual([+ [join("/tmp/hf-timeline-bound-compiled", REMOTE_MEDIA_SUBDIR)],+ ]);+ });});describe("video extraction source logging", () => {diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.tsindex 994249f572..877b602cfd 100644--- a/packages/producer/src/services/render/stages/extractVideosStage.ts+++ b/packages/producer/src/services/render/stages/extractVideosStage.ts@@ -56,6 +56,7 @@ import {type RenderJob,} from "../../renderOrchestrator.js";import { materializeExtractedFramesForCompiledDir, type CompositionMetadata } from "../shared.js";+import { REMOTE_MEDIA_SUBDIR } from "../../htmlCompiler.js";import { resolveRenderFpsConfig } from "../../fileServer.js";import type { ProducerLogger } from "../../../logger.js";import { encoderFailureError } from "../encoderInterruption.js";@@ -504,6 +505,7 @@ export async function runExtractVideosStage(maxTransientRetries: extractionPolicy.maxTransientRetries,collectProbeFailures: extractionPolicy.failureMode === "enforce",deferRangeExtraction,+ contentKeyedDirs: [join(compiledDir, REMOTE_MEDIA_SUBDIR)],},abortSignal,cfg,