Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): show a scene's starting state on a render's first frame (#5614)
Miguel Ángel8c00ebad80c6
diff --git a/packages/core/src/runtime/adapters/gsap.steps.test.ts b/packages/core/src/runtime/adapters/gsap.steps.test.tsindex 68a2920659..a04a1e5d9c 100644--- a/packages/core/src/runtime/adapters/gsap.steps.test.ts+++ b/packages/core/src/runtime/adapters/gsap.steps.test.ts@@ -1,7 +1,7 @@// @vitest-environment happy-domimport gsap from "gsap";import { describe, expect, it } from "vitest";-import { createGsapAdapter, rerenderGsapTimelineAt } from "./gsap";+import { createGsapAdapter, rerenderGsapTimelineAt, seekGsapTimeline } from "./gsap";import type { RuntimeTimelineLike } from "../types";// Every 0.1 s: hide all three frames, then show one, as frame-by-frame films do.@@ -531,3 +531,51 @@ describe("gsap adapter just after a tween's start", () => {}});});++describe("seekGsapTimeline", () => {+ function hiddenAtZero() {+ const word = document.body.appendChild(document.createElement("span"));+ let calls = 0;+ const timeline = gsap.timeline({ paused: true });+ timeline.set(word, { visibility: "hidden" }, 0);+ timeline.call(() => void calls++, [], 0);+ timeline.set(word, { visibility: "visible" }, 0.5);+ return { word, timeline: timeline as unknown as RuntimeTimelineLike, calls: () => calls };+ }++ it("renders a never-rendered timeline seeked to the time it sits at", () => {+ const film = hiddenAtZero();+ seekGsapTimeline(film.timeline, 0, false);+ expect(film.word.style.visibility).toBe("hidden");+ });++ it("redraws a timeline parked at one time once, not on every seek", () => {+ const film = hiddenAtZero();+ const real = film.timeline;+ let moves = 0;+ const counted = {+ ...real,+ time: () => real.time(),+ totalTime: (time?: number, suppressEvents?: boolean) => {+ if (time === undefined) return real.totalTime!();+ moves++;+ return real.totalTime!(time, suppressEvents);+ },+ getChildren: real.getChildren!.bind(real),+ } as RuntimeTimelineLike;+ seekGsapTimeline(counted, 0, false);+ const firstSeekMoves = moves;+ seekGsapTimeline(counted, 0, false);+ seekGsapTimeline(counted, 0, false);+ expect(firstSeekMoves).toBeGreaterThan(1);+ expect(moves - firstSeekMoves).toBe(2);+ });++ it("fires a call at that time once across the next seeks", () => {+ const film = hiddenAtZero();+ seekGsapTimeline(film.timeline, 0, false);+ seekGsapTimeline(film.timeline, 0, false);+ seekGsapTimeline(film.timeline, 0.1, false);+ expect(film.calls()).toBe(1);+ });+});diff --git a/packages/core/src/runtime/adapters/gsap.ts b/packages/core/src/runtime/adapters/gsap.tsindex e152b986d8..b2c7b5cd45 100644--- a/packages/core/src/runtime/adapters/gsap.ts+++ b/packages/core/src/runtime/adapters/gsap.ts@@ -49,6 +49,37 @@ export function rerenderGsapTimelineAt(}}+type GsapRenderState = { _initted?: number };++function gsapNeverRendered(timeline: RuntimeTimelineLike): boolean {+ const [child] = (timeline.getChildren?.(false, true, true) ?? []) as { parent?: unknown }[];+ const rendered = child?.parent as GsapRenderState | undefined;+ return rendered !== undefined && !rendered._initted;+}++export function seekGsapTimeline(+ timeline: RuntimeTimelineLike,+ t: number,+ suppressEvents: boolean,+): boolean {+ if (typeof timeline.totalTime !== "function") {+ timeline.seek(t, suppressEvents);+ return false;+ }+ const gsapSkipsRenderInPlace = Number(timeline.totalTime()) === t;+ timeline.totalTime(t, suppressEvents);+ if (!gsapSkipsRenderInPlace || !gsapNeverRendered(timeline)) return false;+ rerenderGsapTimelineAt(+ {+ totalTime: timeline.totalTime.bind(timeline),+ totalDuration: timeline.totalDuration?.bind(timeline),+ getChildren: timeline.getChildren?.bind(timeline),+ },+ t,+ );+ return true;+}+type GsapParent = {_dur: number;_tDur: number;diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.tsindex abac71baef..40a6b06a48 100644--- a/packages/core/src/runtime/init.test.ts+++ b/packages/core/src/runtime/init.test.ts@@ -3119,7 +3119,7 @@ describe("initSandboxRuntimeModular", () => {const rootTimeline = createMockTimeline(10);const originalTotalTime = rootTimeline.totalTime;rootTimeline.totalTime = (time: number, suppressEvents?: boolean) => {- seekCalls.push({ time, suppressEvents });+ if (time !== undefined) seekCalls.push({ time, suppressEvents });return originalTotalTime?.(time, suppressEvents);};@@ -3155,7 +3155,7 @@ describe("initSandboxRuntimeModular", () => {const rootTimeline = createMockTimeline(10);const originalTotalTime = rootTimeline.totalTime;rootTimeline.totalTime = (time: number, suppressEvents?: boolean) => {- seekCalls.push({ time, suppressEvents });+ if (time !== undefined) seekCalls.push({ time, suppressEvents });return originalTotalTime?.(time, suppressEvents);};Object.assign(rootTimeline, {@@ -3177,6 +3177,26 @@ describe("initSandboxRuntimeModular", () => {expect(seekCalls).toEqual([{ time: 2, suppressEvents: false }]);});+ it("keeps a later root tween's value over a nested scene the root already drew", () => {+ document.body.innerHTML = `<div data-composition-id="main" data-root="true" data-start="0" data-duration="2" data-width="1920" data-height="1080"><div data-composition-id="child" data-start="0" data-duration="2"><div id="box"></div></div></div>`;+ const box = document.getElementById("box")!;+ const options = { duration: 2, ease: "none", immediateRender: false, lazy: false };+ const child = gsap.timeline({ smoothChildTiming: false });+ child.fromTo(box, { opacity: 0 }, { ...options, opacity: 1, overwrite: false }, 0);+ const main = gsap.timeline({ paused: true, smoothChildTiming: false });+ main.add(child, 0);+ main.fromTo(box, { opacity: 0.8 }, { ...options, opacity: 1, overwrite: false }, 0);+ window.__timelines = { main, child };+ initSandboxRuntimeModular();++ window.__player?.renderSeek(1);+ expect(Number(box.style.opacity)).toBeCloseTo(0.9, 5);++ child.set({}, {}, 1.5);+ window.__player?.renderSeek(1);+ expect(Number(box.style.opacity)).toBeCloseTo(0.9, 5);+ });+it("fires a call added after the first seek exactly once", () => {const root = document.createElement("div");root.setAttribute("data-composition-id", "main");diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex d85e18a6a5..0dc4ad4ba4 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -16,7 +16,7 @@ import {import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";import { injectCompositionCssVariables } from "./getVariables";import { createCssAdapter } from "./adapters/css";-import { createGsapAdapter, rerenderGsapTimelineAt } from "./adapters/gsap";+import { createGsapAdapter, rerenderGsapTimelineAt, seekGsapTimeline } from "./adapters/gsap";import { createAnimeJsAdapter } from "./adapters/animejs";import { createLottieAdapter } from "./adapters/lottie";import { createThreeAdapter } from "./adapters/three";@@ -1663,8 +1663,7 @@ export function initSandboxRuntimeModular(): void {// Without this, children added after the root was created may still show initial styles.if (autoNestedChildren.length > 0) {try {- const currentTime = rootTimeline.time();- rootTimeline.seek(currentTime, false); // false = don't suppress events+ seekGsapTimeline(rootTimeline, rootTimeline.time(), false);} catch {/* ignore */}@@ -2323,7 +2322,7 @@ export function initSandboxRuntimeModular(): void {// abort restoring the transport time below (the catch would swallow it too).pauseTimelineIfPossible(state.capturedTimeline);if (typeof state.capturedTimeline.seek === "function") {- state.capturedTimeline.seek(previousTime, false);+ seekGsapTimeline(state.capturedTimeline, previousTime, false);}} catch (err) {// keep runtime resilient even if a timeline implementation throws@@ -4240,11 +4239,7 @@ export function initSandboxRuntimeModular(): void {const suppressEvents = options?.suppressEvents === true;// Guarded: a partial timeline without pause() must still get its seek.pauseTimelineIfPossible(timeline);- if (typeof timeline.totalTime === "function") {- timeline.totalTime(timeSeconds, suppressEvents);- } else {- timeline.seek(timeSeconds, suppressEvents);- }+ seekGsapTimeline(timeline, timeSeconds, suppressEvents);} catch (err) {swallow(swallowLabel, err);}@@ -4399,8 +4394,8 @@ export function initSandboxRuntimeModular(): void {}try {if (typeof tl.totalTime === "function") {- tl.totalTime(tlSeekTime, suppressEvents);- if (!suppressEvents && !hasZeroDurationCallbackTween(tl)) {+ const redrawn = seekGsapTimeline(tl, tlSeekTime, suppressEvents);+ if (!redrawn && !suppressEvents && !hasZeroDurationCallbackTween(tl)) {// The first seek is the only eventful one; the re-render only refreshes styles.rerenderGsapTimelineAt({diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjsindex 0bf0e4f5d2..3bf6f6b093 100644--- a/packages/producer/scripts/test-classification.mjs+++ b/packages/producer/scripts/test-classification.mjs@@ -30,6 +30,7 @@ const INTEGRATION_TEST_FILES = new Set(["src/services/assetMediaType.test.ts","src/services/htmlCompiler.mediaType.test.ts","src/services/htmlCompiler.naturalDuration.test.ts",+ "src/services/render/firstFrameStartingState.integration.test.ts","src/services/render/hlsRender.integration.test.ts","src/services/render/motionBlurVideo.integration.test.ts","src/services/render/renderCancel.integration.test.ts",diff --git a/packages/producer/src/services/render/firstFrameStartingState.integration.test.ts b/packages/producer/src/services/render/firstFrameStartingState.integration.test.tsnew file mode 100644index 0000000000..0e2c35788c--- /dev/null+++ b/packages/producer/src/services/render/firstFrameStartingState.integration.test.ts@@ -0,0 +1,114 @@+// Frame 0 of a real render shows the starting state that a sub-composition's timeline sets at+// t=0, not the page's pre-timeline styles. Needs Chrome + ffmpeg: integration lane.++import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join, resolve } from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { resolveConfig } from "@hyperframes/engine";+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";+import { HAS_FFMPEG, frameLevels } from "./frameLevelsTestFixtures.js";++const WIDTH = 160;+const HEIGHT = 90;+const FPS = 30;+const REVEAL_FRAME = 15;++const INDEX = `<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8" />+ <script src="assets/gsap.min.js"></script>+ <style>* { margin: 0; padding: 0; } body { overflow: hidden; background: #000; }</style>+</head>+<body>+ <div id="root" data-composition-id="main" data-start="0" data-duration="1"+ data-width="${WIDTH}" data-height="${HEIGHT}" data-fps="${FPS}">+ <div id="s-words" data-composition-id="words" data-composition-src="compositions/words.html"+ data-start="0" data-duration="1" data-track-index="0"+ data-width="${WIDTH}" data-height="${HEIGHT}"></div>+ </div>+ <script>window.__timelines["main"] = gsap.timeline({ paused: true });</script>+</body>+</html>+`;++// The words have no hidden CSS state: only the timeline's t=0 steps hide them.+const WORDS = `<!DOCTYPE html>+<html>+<body>+<template id="words-template">+ <style>+ @font-face { font-family: "Fixture"; src: url("assets/fixture.woff2") format("woff2"); font-weight: 500; }+ #words { position: absolute; inset: 0; background: #000; }+ #words .w { position: absolute; top: 20px; font: 500 40px "Fixture", serif; color: #fff; }+ </style>+ <div id="words" data-composition-id="words" data-width="${WIDTH}" data-height="${HEIGHT}" data-duration="1">+ <span class="w">Aa</span><span class="w">Bb</span>+ </div>+ <script>+ (() => {+ const tl = gsap.timeline({ paused: true });+ const words = [...document.querySelectorAll("#words .w")];+ words.forEach((w) => tl.set(w, { visibility: "hidden" }, 0));+ words.forEach((w, i) => tl.set(w, { visibility: "visible", left: 10 + i * 70 }, ${REVEAL_FRAME / FPS}));+ tl.set({}, {}, 1);+ window.__timelines["words"] = tl;+ })();+ </script>+</template>+</body>+</html>+`;++let root: string;++describe.skipIf(!HAS_FFMPEG)(+ "first frame of a real render — sub-composition starting state",+ () => {+ let levels: number[];++ beforeAll(async () => {+ root = mkdtempSync(join(tmpdir(), "hf-first-frame-"));+ mkdirSync(join(root, "assets"), { recursive: true });+ mkdirSync(join(root, "compositions"), { recursive: true });+ copyFileSync(+ resolve(import.meta.dirname, "../../../../core/node_modules/gsap/dist/gsap.min.js"),+ join(root, "assets/gsap.min.js"),+ );+ copyFileSync(+ resolve(+ import.meta.dirname,+ "../../../tests/heygen-promo-preview-assets/src/TT_Norms_Pro_Medium.woff2",+ ),+ join(root, "assets/fixture.woff2"),+ );+ writeFileSync(join(root, "index.html"), INDEX);+ writeFileSync(join(root, "compositions/words.html"), WORDS);+ const job = createRenderJob({+ fps: FPS,+ quality: "draft",+ format: "mp4",+ hdrMode: "force-sdr",+ workers: 1,+ producerConfig: resolveConfig({ browserGpuMode: "software" }),+ });+ const output = join(root, "out.mp4");+ await executeRenderJob(job, root, output);+ levels = frameLevels(output, WIDTH, HEIGHT);+ }, 120_000);++ afterAll(() => {+ if (root) rmSync(root, { recursive: true, force: true });+ });++ it("frame 0 matches frame 1: the words are already hidden", () => {+ expect(levels[0]).toBeCloseTo(levels[1]!, 0);+ expect(levels[0]).toBeLessThan(1);+ });++ it("the words appear when the timeline reveals them", () => {+ expect(levels[REVEAL_FRAME]! - levels[0]!).toBeGreaterThan(2);+ });+ },+);diff --git a/packages/producer/src/services/render/frameLevelsTestFixtures.ts b/packages/producer/src/services/render/frameLevelsTestFixtures.tsnew file mode 100644index 0000000000..7f89021e4c--- /dev/null+++ b/packages/producer/src/services/render/frameLevelsTestFixtures.ts@@ -0,0 +1,23 @@+import { spawnSync } from "node:child_process";+import { getFfmpegBinary } from "@hyperframes/engine";++export const FFMPEG = getFfmpegBinary();+export const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0;++/** Mean grey level of every frame of an encoded video, 0-255. */+export function frameLevels(videoPath: string, width: number, height: number): number[] {+ const raw = spawnSync(+ FFMPEG,+ ["-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-"],+ { maxBuffer: 64 * 1024 * 1024 },+ );+ if (raw.status !== 0) throw new Error(`decode failed: ${raw.stderr}`);+ const frameBytes = width * height;+ const levels: number[] = [];+ for (let offset = 0; offset + frameBytes <= raw.stdout.length; offset += frameBytes) {+ let sum = 0;+ for (let i = offset; i < offset + frameBytes; i++) sum += raw.stdout[i]!;+ levels.push(sum / frameBytes);+ }+ return levels;+}diff --git a/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts b/packages/producer/src/services/render/segmentedColorGrading.integration.test.tsindex 57356f9698..33d3012daf 100644--- a/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts+++ b/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts@@ -7,11 +7,9 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { afterAll, beforeAll, describe, expect, it } from "vitest";-import { getFfmpegBinary, resolveConfig } from "@hyperframes/engine";+import { resolveConfig } from "@hyperframes/engine";import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";--const FFMPEG = getFfmpegBinary();-const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0;+import { FFMPEG, HAS_FFMPEG, frameLevels } from "./frameLevelsTestFixtures.js";const WIDTH = 160;const HEIGHT = 90;@@ -55,26 +53,6 @@ const COMPOSITION = `<!DOCTYPE html>let root: string;-/** Mean RGB level of every frame of an encoded video, 0-255. */-function frameLevels(videoPath: string): number[] {- const raw = spawnSync(- FFMPEG,- ["-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-"],- {- maxBuffer: 64 * 1024 * 1024,- },- );- if (raw.status !== 0) throw new Error(`decode failed: ${raw.stderr}`);- const frameBytes = WIDTH * HEIGHT;- const levels: number[] = [];- for (let offset = 0; offset + frameBytes <= raw.stdout.length; offset += frameBytes) {- let sum = 0;- for (let i = offset; i < offset + frameBytes; i++) sum += raw.stdout[i]!;- levels.push(sum / frameBytes);- }- return levels;-}-interface Render {levels: number[];phases: string[];@@ -112,7 +90,7 @@ async function render(name: string, segmented: boolean, workers: number): Promistry {const output = join(root, `${name}.mp4`);await executeRenderJob(job, root, output);- return { levels: frameLevels(output), phases, workerCount };+ return { levels: frameLevels(output, WIDTH, HEIGHT), phases, workerCount };} finally {if (saved.capture === undefined) delete process.env.HF_SEGMENTED_CAPTURE;else process.env.HF_SEGMENTED_CAPTURE = saved.capture;diff --git a/scripts/test-reachability.json b/scripts/test-reachability.jsonindex bbf5feddd9..621c2140fb 100644--- a/scripts/test-reachability.json+++ b/scripts/test-reachability.json@@ -61,7 +61,7 @@],"sources": {"packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7",- "packages/producer/scripts/test-classification.mjs": "11102d43fdae5ab611c2f840e3d7fb4f367683fbddf9dfc770e0a623deb6aef2",+ "packages/producer/scripts/test-classification.mjs": "18ec188edb061f2a729d261a64fb22682064e4f438413fdeb4d502e709cc8684",".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864","packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1","packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",