Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): copy border-width/style/color onto the replacement video frame (#3993)
miga-heygene23b3a3e31d8
diff --git a/packages/core/src/inline-scripts/parityContract.ts b/packages/core/src/inline-scripts/parityContract.tsindex 785ea7b525..1d733c9875 100644--- a/packages/core/src/inline-scripts/parityContract.ts+++ b/packages/core/src/inline-scripts/parityContract.ts@@ -14,6 +14,9 @@ export const MEDIA_VISUAL_STYLE_PROPERTIES = ["filter","mix-blend-mode","backdrop-filter",+ "border-width",+ "border-style",+ "border-color","border-radius","overflow","clip-path",diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.tsindex f7916e93a1..014eb8826d 100644--- a/packages/core/src/runtime/colorGrading.test.ts+++ b/packages/core/src/runtime/colorGrading.test.ts@@ -237,6 +237,18 @@ describe("createColorGradingRuntime", () => {},);+ it("draws a bordered source's grading canvas without a border over its whole box", () => {+ vi.stubGlobal("devicePixelRatio", 1);+ const video = makeDrawableVideo();+ video.style.border = "16px solid rgb(255, 0, 0)";++ const { canvas } = startRuntimeWithVideo(video);++ expect(canvas.style.borderStyle).toBe("none");+ expect(canvas.width).toBe(640);+ expect(canvas.height).toBe(360);+ });+it("says an element is graded only while it draws through a grading canvas", () => {const { video } = startRuntimeWithVideo();const plain = document.createElement("video");@@ -875,6 +887,40 @@ describe("createColorGradingRuntime", () => {expect(canvas.style.opacity).toBe("0.75");});+ it("drops the injected render frame's border while grading draws over it", () => {+ const video = makeDrawableVideo();+ Object.defineProperty(video, "readyState", {+ value: HTMLMediaElement.HAVE_METADATA,+ configurable: true,+ });+ Object.defineProperty(video, "videoWidth", { value: 0, configurable: true });+ Object.defineProperty(video, "videoHeight", { value: 0, configurable: true });+ document.body.appendChild(video);+ runtime = createColorGradingRuntime();+ const canvas = document.querySelector<HTMLCanvasElement>("[data-hf-color-grading-canvas]");+ if (!canvas) throw new Error("Expected color grading canvas");++ const frame = document.createElement("img");+ frame.id = "__render_frame_hero-video__";+ frame.className = "__render_frame__";+ frame.style.border = "16px solid rgb(255, 0, 0)";+ Object.defineProperty(frame, "complete", { value: true, configurable: true });+ Object.defineProperty(frame, "naturalWidth", { value: 640, configurable: true });+ Object.defineProperty(frame, "naturalHeight", { value: 360, configurable: true });+ video.parentNode?.insertBefore(frame, canvas);+ video.style.setProperty("visibility", "hidden", "important");++ runtime.redraw();++ expect(canvas.style.display).toBe("block");+ expect(frame.style.borderStyle).toBe("none");++ video.style.border = "16px dashed rgb(255, 0, 0)";+ video.removeAttribute(HF_COLOR_GRADING_ATTR);+ runtime.refresh();+ expect(frame.style.borderStyle).toBe("dashed");+ });+it("keeps a staged scene copy's canvas on its own render frame (#3994)", async () => {// Page-side shader transitions clone the scene, ids included, into a staging// layer. When both videos paired with the live frame, their canvases tradeddiff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.tsindex f2b932afb7..f0c5ae4a32 100644--- a/packages/core/src/runtime/colorGrading.ts+++ b/packages/core/src/runtime/colorGrading.ts@@ -210,6 +210,7 @@ interface ColorGradingEntry extends ColorGradingRenderer {touchedParent: HTMLElement | null;parentInlinePosition: string | null;sourceHidden: boolean;+ borderlessFrame: HTMLElement | null;sourceInlineOpacity: string | null;sourceInlineOpacityPriority: string;sourceOpacityForCanvas: string;@@ -1947,7 +1948,21 @@ function replaceProgramResources(entry: ColorGradingEntry): boolean {return true;}+function hideFrameBorder(entry: ColorGradingEntry, frame: HTMLElement): void {+ if (entry.borderlessFrame !== frame) restoreFrameBorder(entry);+ entry.borderlessFrame = frame;+ frame.style.borderStyle = "none";+}++function restoreFrameBorder(entry: ColorGradingEntry): void {+ const frame = entry.borderlessFrame;+ if (!frame) return;+ entry.borderlessFrame = null;+ frame.style.borderStyle = window.getComputedStyle(entry.element).borderStyle;+}+function restoreSourceElement(entry: ColorGradingEntry): void {+ restoreFrameBorder(entry);if (!entry.sourceHidden) return;entry.element.removeAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR);const opacity = entry.element.style.getPropertyValue("opacity");@@ -2646,6 +2661,7 @@ function updateCanvasLayout(const computed = window.getComputedStyle(styleSource);copyMediaVisualStyles(canvas.style, computed);+ canvas.style.borderStyle = "none";canvas.style.pointerEvents = "none";canvas.style.position = "absolute";canvas.style.inset = "auto";@@ -3182,6 +3198,7 @@ function drawEntry(entry: ColorGradingEntry): boolean {);drawFullscreenQuad(gl, program);hideSourceElement(entry);+ if (injectedFrameSource) hideFrameBorder(entry, source);entry.hasDrawn = true;entry.drawError = null;return true;@@ -3584,6 +3601,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {touchedParent: null,parentInlinePosition: null,sourceHidden: false,+ borderlessFrame: null,sourceInlineOpacity: null,sourceInlineOpacityPriority: "",sourceOpacityForCanvas: window.getComputedStyle(element).opacity || "1",diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.tsindex 0a622eb256..10ba55fc1f 100644--- a/packages/engine/src/services/frameCapture.ts+++ b/packages/engine/src/services/frameCapture.ts@@ -1061,10 +1061,9 @@ async function initDrawElementOrTransparentBackground(const cssFx = await detectCssEffectRisk(page);if (cssFx) {session.deGateReason = `css_effect:${(cssFx.split(":")[0] ?? "").replace(/[^a-z-]/gi, "")}`;- // Full specific effect ("filter:blur" / "filter:drop-shadow" /- // "backdrop-filter" / "clip-path") — `deGateReason` sanitizes- // this to the low-cardinality prefix; `deFallbackTrigger` keeps- // the fine-grained value for the diagnostic profile emission.+ // Full specific effect from `detectCssEffectRisk` (e.g. "filter:blur",+ // "mix-blend-mode", "webgl-context") — `deGateReason` keeps its+ // low-cardinality prefix, `deFallbackTrigger` the full value.session.deFallbackTrigger = cssFx;console.log(`[engine] fast capture: falling back to ${session.launchCaptureMode} capture — ` +diff --git a/packages/engine/src/services/screenshotService.ts b/packages/engine/src/services/screenshotService.tsindex 0a9784fbc5..0ade72519a 100644--- a/packages/engine/src/services/screenshotService.ts+++ b/packages/engine/src/services/screenshotService.ts@@ -735,6 +735,15 @@ export async function injectVideoFramesBatch(? 1: opacityParsed;+ // Measure first: an in-flow bordered <img> sibling would shrink the video's flex box.+ const videoRect = video.getBoundingClientRect();+ const videoBox = {+ left: Number.isFinite(video.offsetLeft) ? video.offsetLeft : 0,+ top: Number.isFinite(video.offsetTop) ? video.offsetTop : 0,+ width: video.offsetWidth > 0 ? video.offsetWidth : videoRect.width,+ height: video.offsetHeight > 0 ? video.offsetHeight : videoRect.height,+ };+if (isNewImage) {img = document.createElement("img");img.classList.add("__render_frame__");@@ -770,24 +779,16 @@ export async function injectVideoFramesBatch(// instead of flowing below it. With position:relative, both elements// stack vertically — the <img> lands below the video and gets clipped// by any overflow:hidden ancestor (e.g., border-radius wrappers).- //- // Apply this after visual style copying so the measured used box is- // the final authority for replacement frame geometry.- {- const videoRect = video.getBoundingClientRect();- const offsetLeft = Number.isFinite(video.offsetLeft) ? video.offsetLeft : 0;- const offsetTop = Number.isFinite(video.offsetTop) ? video.offsetTop : 0;- const offsetWidth = video.offsetWidth > 0 ? video.offsetWidth : videoRect.width;- const offsetHeight = video.offsetHeight > 0 ? video.offsetHeight : videoRect.height;- img.style.position = "absolute";- img.style.inset = "auto";- img.style.left = `${offsetLeft}px`;- img.style.top = `${offsetTop}px`;- img.style.right = "auto";- img.style.bottom = "auto";- img.style.width = `${offsetWidth}px`;- img.style.height = `${offsetHeight}px`;- }+ img.style.position = "absolute";+ img.style.inset = "auto";+ img.style.left = `${videoBox.left}px`;+ img.style.top = `${videoBox.top}px`;+ img.style.right = "auto";+ img.style.bottom = "auto";+ img.style.width = `${videoBox.width}px`;+ img.style.height = `${videoBox.height}px`;+ // `videoBox` is a border-box even when the video is content-box.+ img.style.boxSizing = "border-box";img.style.objectFit = computedStyle.objectFit;img.style.objectPosition = computedStyle.objectPosition;img.style.zIndex = computedStyle.zIndex;diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjsindex 903bfdf2ed..3c3969bca9 100644--- a/packages/producer/scripts/test-classification.mjs+++ b/packages/producer/scripts/test-classification.mjs@@ -34,6 +34,7 @@ const INTEGRATION_TEST_FILES = new Set(["src/services/render/renderCancel.integration.test.ts","src/services/render/stages/compileStage.mediaType.test.ts","src/services/render/stages/gifEncodeArgs.test.ts",+ "src/services/videoFrameBorderClip.test.ts","src/utils/audioRegression.test.ts","src/utils/streamDurationParity.test.ts",]);diff --git a/packages/producer/src/services/videoFrameBorderClip.test.ts b/packages/producer/src/services/videoFrameBorderClip.test.tsnew file mode 100644index 0000000000..a620716521--- /dev/null+++ b/packages/producer/src/services/videoFrameBorderClip.test.ts@@ -0,0 +1,180 @@+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";+import puppeteer, { type Browser, type Page } from "puppeteer";+import { injectVideoFramesBatch } from "@hyperframes/engine";++/** Colour channels of one screenshot pixel. Alpha is always opaque here, so it is dropped. */+type Rgb = { r: number; g: number; b: number };++/** A probe coordinate in screenshot space. */+type Point = readonly [x: number, y: number];++const VIDEO_ID = "v1";++// Solid blue 1x1 PNG, stretched over the whole video box by the UA default+// `object-fit: fill` — makes injected video content trivially distinguishable+// from the white page background and the red border in a screenshot.+const BLUE_PIXEL_DATA_URI =+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYPj/HwADAgH/5ncLrgAAAABJRU5ErkJggg==";++const BOX_WIDTH = 200;+const BOX_HEIGHT = 120;+const BORDER_WIDTH = 8;+const CORNER_RADIUS = 24;++/** Mid-way down the left border edge, far below the rounded corner's arc. */+const BORDER_EDGE: Point = [BORDER_WIDTH / 2, BOX_HEIGHT / 2];+/** Dead centre of the box, well inside the injected frame's content. */+const VIDEO_CENTER: Point = [BOX_WIDTH / 2, BOX_HEIGHT / 2];+/** Inside the corner that both the border-radius curve and the clip-path inset remove. */+const CLIPPED_CORNER: Point = [1, 1];++async function readPixel(page: Page, screenshotBase64: string, [x, y]: Point): Promise<Rgb> {+ return page.evaluate(+ async (dataUri, probeX, probeY) => {+ const img = new Image();+ img.src = dataUri;+ await img.decode();+ const canvas = document.createElement("canvas");+ canvas.width = img.naturalWidth;+ canvas.height = img.naturalHeight;+ const ctx = canvas.getContext("2d");+ if (!ctx) throw new Error("2d context unavailable");+ ctx.drawImage(img, 0, 0);+ const [r, g, b] = ctx.getImageData(probeX, probeY, 1, 1).data;+ if (r === undefined || g === undefined || b === undefined) {+ throw new Error(`no pixel data at (${probeX}, ${probeY})`);+ }+ return { r, g, b };+ },+ `data:image/png;base64,${screenshotBase64}`,+ x,+ y,+ );+}++/** Renders the styled `<video>`, substitutes its frame, and returns a base64 PNG screenshot. */+async function captureStyledVideoFrame(page: Page, videoAttributes: string): Promise<string> {+ await page.setContent(`<!doctype html>+ <style>html, body { margin: 0; padding: 0; background: #ffffff; }</style>+ <video+ id="${VIDEO_ID}"+ class="clip"+ muted+ ${videoAttributes}+ style="position:absolute;left:0;top:0;width:${BOX_WIDTH}px;height:${BOX_HEIGHT}px;object-fit:fill;+ border:${BORDER_WIDTH}px solid red;border-radius:${CORNER_RADIUS}px;+ clip-path:inset(0 round ${CORNER_RADIUS}px);"+ ></video>`);+ await injectVideoFramesBatch(page, [{ videoId: VIDEO_ID, dataUri: BLUE_PIXEL_DATA_URI }]);+ return page.screenshot({ type: "png", encoding: "base64" });+}++function expectRedBorder(pixel: Rgb): void {+ expect(pixel.r).toBeGreaterThan(180);+ expect(pixel.b).toBeLessThan(80);+}++function expectBlueVideoContent(pixel: Rgb): void {+ expect(pixel.b).toBeGreaterThan(180);+}++function expectWhitePageBackground(pixel: Rgb): void {+ expect(pixel.r).toBeGreaterThan(240);+ expect(pixel.g).toBeGreaterThan(240);+ expect(pixel.b).toBeGreaterThan(240);+}++describe("video border/border-radius/clip-path on the replacement render frame", () => {+ let browser: Browser;+ let page: Page;++ beforeAll(async () => {+ browser = await puppeteer.launch({+ headless: true,+ args: ["--no-sandbox", "--disable-setuid-sandbox"],+ });+ }, 30_000);++ beforeEach(async () => {+ page = await browser.newPage();+ await page.setViewport({ width: BOX_WIDTH, height: BOX_HEIGHT });+ });++ afterEach(async () => {+ await page?.close();+ });++ afterAll(async () => {+ await browser?.close();+ });++ // The border must reach the replacement <img>, not just the hidden <video>;+ // without border-width/style/color on the copy list the frame paints through it.+ it("paints a static border and clips to border-radius/clip-path on an untimed <video class=clip>", async () => {+ const screenshot = await captureStyledVideoFrame(page, "");++ expectRedBorder(await readPixel(page, screenshot, BORDER_EDGE));+ expectBlueVideoContent(await readPixel(page, screenshot, VIDEO_CENTER));+ expectWhitePageBackground(await readPixel(page, screenshot, CLIPPED_CORNER));+ });++ it("paints the same border/radius/clip-path on a timed <video data-start> clip", async () => {+ const screenshot = await captureStyledVideoFrame(page, 'data-start="0" data-duration="5"');++ expectRedBorder(await readPixel(page, screenshot, BORDER_EDGE));+ expectBlueVideoContent(await readPixel(page, screenshot, VIDEO_CENTER));+ expectWhitePageBackground(await readPixel(page, screenshot, CLIPPED_CORNER));+ });++ // A bordered <img> sibling still in flow shrinks a flex row's video, so the+ // video must be measured before the <img> is styled (the style-9-prod layout).+ it.each(["border-box", "content-box"])(+ "keeps the replacement <img>'s box identical to a %s video's own box in a flex-centered layout",+ async (boxSizing) => {+ const FLEX_ROW_WIDTH = 500;+ const FLEX_ROW_HEIGHT = 300;++ await page.setViewport({ width: FLEX_ROW_WIDTH, height: FLEX_ROW_HEIGHT });+ await page.setContent(`<!doctype html>+ <style>+ * { box-sizing: ${boxSizing}; }+ html, body { margin: 0; padding: 0; background: #ffffff; }+ #aroll-container {+ display: flex; justify-content: center; align-items: center;+ width: ${FLEX_ROW_WIDTH}px; height: ${FLEX_ROW_HEIGHT}px;+ }+ </style>+ <div id="aroll-container">+ <video+ id="${VIDEO_ID}"+ class="clip"+ muted+ style="width:100%;height:150px;object-fit:fill;+ border:${BORDER_WIDTH}px solid red;border-radius:${CORNER_RADIUS}px;"+ ></video>+ </div>`);++ await injectVideoFramesBatch(page, [{ videoId: VIDEO_ID, dataUri: BLUE_PIXEL_DATA_URI }]);++ const { videoBox, imgBox } = await page.evaluate((videoId) => {+ const video = document.getElementById(videoId);+ const img = video?.nextElementSibling;+ if (!video || !img) throw new Error(`no replacement frame beside #${videoId}`);+ const roundedRect = (rect: DOMRect) => ({+ left: Math.round(rect.left),+ top: Math.round(rect.top),+ width: Math.round(rect.width),+ height: Math.round(rect.height),+ });+ return {+ videoBox: roundedRect(video.getBoundingClientRect()),+ imgBox: roundedRect(img.getBoundingClientRect()),+ };+ }, VIDEO_ID);++ expect(imgBox).toEqual(videoBox);+ // Confirms the fixture gave the flex row room to shrink into.+ if (boxSizing === "border-box") expect(videoBox.width).toBe(FLEX_ROW_WIDTH);+ },+ );+});diff --git a/packages/producer/tests/style-9-prod/output/output.mp4 b/packages/producer/tests/style-9-prod/output/output.mp4index 6d2e3fa8fa..ee93193fbc 100644--- a/packages/producer/tests/style-9-prod/output/output.mp4+++ b/packages/producer/tests/style-9-prod/output/output.mp4@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:d8fb3bbb1666443c7f004a7b9c6dfba594097611fed18cf386e91fc45f5d378b-size 13520567+oid sha256:586aef50c55b7cc03d02b5c2f4200fd0a13453f1d09fbfe1ab221b0c71b88336+size 13670413diff --git a/scripts/test-reachability.json b/scripts/test-reachability.jsonindex 72d41c1242..9ac2e6c265 100644--- a/scripts/test-reachability.json+++ b/scripts/test-reachability.json@@ -61,7 +61,7 @@],"sources": {"packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7",- "packages/producer/scripts/test-classification.mjs": "d7ca46d663bfa7012147d1eff63a9e599f44393d7d65fd910d5b96cceb2b2f44",+ "packages/producer/scripts/test-classification.mjs": "5cf0848da08f0fd2c40521d1556cfdb00b5a072cbd0ea0bc54945990abb709e4",".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864","packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1","packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",