Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): stop small previews dropping layers on heavy compositions (#5268)
Miguel Ángel5562c280daaa
diff --git a/.github/workflows/player-perf.yml b/.github/workflows/player-perf.ymlindex 811f9a41ac..5d6d73fa24 100644--- a/.github/workflows/player-perf.yml+++ b/.github/workflows/player-perf.yml@@ -124,6 +124,13 @@ jobs:PUPPETEER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}run: bun run test:browser-security+ - name: Verify the preview rasters at its shown size (load shard only)+ if: matrix.shard == 'load'+ working-directory: packages/player+ env:+ PUPPETEER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}+ run: bun run test:browser-raster+- name: Run player perf — ${{ matrix.shard }} (measure mode)working-directory: packages/playerenv:diff --git a/packages/core/src/runtime/init.swapScenes.test.ts b/packages/core/src/runtime/init.swapScenes.test.tsindex dfb37af6dd..c8191865e7 100644--- a/packages/core/src/runtime/init.swapScenes.test.ts+++ b/packages/core/src/runtime/init.swapScenes.test.ts@@ -691,6 +691,13 @@ describe("__hfSwapScenes", () => {},],["the opacity stamp", '<video src="clip.mp4"></video>', () => {}, "data-hf-authored-opacity"],+ [+ "the preview raster mark",+ '<video src="clip.mp4"></video>',+ () => {},+ "data-hf-preview-raster",+ ],+ ["the upcoming mark", '<video src="clip.mp4"></video>', () => {}, "data-hf-upcoming"],])("keeps a video on which the runtime wrote %s", async (_, video, arrange, stamp?: string) => {quietMedia();arrange();diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 4ba6043977..776ecdf380 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -1,4 +1,5 @@// fallow-ignore-file code-duplication complexity+import { setPreviewRasterScale } from "./previewRasterHints";import { refreshSvgSelectorAliases } from "../compiler/svgSelectorAliases";import { RUNTIME_FILLER } from "./protocol";import { preloadMedia, releaseMedia, lengthIsAuthored, stopMediaDownload } from "./preloadMedia";@@ -6,7 +7,11 @@ import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFpsimport { instantTolerance } from "../clipFacts";import { isInClipWindow } from "./clipWindow";import { revealTimedClipsAfterFirstPass, SKIPPED_CLIP, skipsHiddenImages } from "./timedClipHide";-import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark";+import {+ PREVIEW_ONLY_ATTRS,+ STUDIO_PREVIEW_LAZY_ATTR,+ STUDIO_PREVIEW_UPCOMING_ATTR,+} from "../studioPreviewMark";import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";import { injectCompositionCssVariables } from "./getVariables";import { createCssAdapter } from "./adapters/css";@@ -280,13 +285,14 @@ const authoredShape = (el: Element): string =>]);// What the runtime writes on a video or audio: style (reset on a kept one), preload, the opacity stamp, a moved-// element's original translate and its own __hf- classes. A proxied src is compared as written; a bound one is not.+// element's original translate, preview marks and its own __hf- classes. A proxied src is compared as written.const RUNTIME_MEDIA_ATTRS = new Set(["style","preload","src",COLOR_GRADING_AUTHORED_OPACITY_ATTR,EDIT_ORIGINAL_TRANSLATE_ATTR,+ ...PREVIEW_ONLY_ATTRS,]);const ownClasses = (value: string) =>value@@ -5024,7 +5030,10 @@ export function initSandboxRuntimeModular(): void {if (state.transportClock) state.transportClock.setRate(state.playbackRate);applyWebAudioRate();},- onSetDisplayScale: setProxyDisplayScale,+ onSetDisplayScale: (scale) => {+ setProxyDisplayScale(scale);+ setPreviewRasterScale(scale);+ },onSetIdleHeartbeat: (slow) => {slowIdleHeartbeat = slow;wakeTransport();diff --git a/packages/core/src/runtime/previewRasterHints.test.ts b/packages/core/src/runtime/previewRasterHints.test.tsnew file mode 100644index 0000000000..c5ef01e714--- /dev/null+++ b/packages/core/src/runtime/previewRasterHints.test.ts@@ -0,0 +1,178 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";+import { setPreviewRasterScale } from "./previewRasterHints";++function layer(style: string, parent: Element = document.body): HTMLElement {+ const element = document.createElement("div");+ element.setAttribute("style", style);+ parent.append(element);+ return element;+}++const POSITIONED = "position: absolute; will-change: transform";+const marked = (element: Element) => element.hasAttribute(PREVIEW_RASTER_ATTR);++afterEach(() => {+ setPreviewRasterScale(1);+ document.body.innerHTML = "";+});++describe("setPreviewRasterScale", () => {+ it("touches nothing until the host reports a scale", () => {+ const words = layer(POSITIONED);+ expect(document.querySelectorAll(`style, [${PREVIEW_RASTER_ATTR}]`)).toHaveLength(0);+ expect(getComputedStyle(words).willChange).toBe("transform");+ });++ it("drops a positioned layer's transform hint while the preview is shown small", () => {+ const words = layer(POSITIONED);+ const word = layer("position: absolute", words);+ const mixed = layer("position: relative; will-change: transform, opacity");+ setPreviewRasterScale(0.275);+ expect(marked(words)).toBe(true);+ expect(marked(word)).toBe(false);+ expect(marked(mixed)).toBe(false);+ expect(getComputedStyle(words).willChange).toBe("auto");+ expect(words.getAttribute("style")).toBe(POSITIONED);+ });++ it("keeps the hint where the stacking context or containing block matters", () => {+ const unpositioned = layer("will-change: transform");+ const holdsFixed = layer(POSITIONED);+ layer("position: fixed", holdsFixed);+ const holdsZIndex = layer(POSITIONED);+ layer("position: relative; z-index: 2", holdsZIndex);+ const holdsBlend = layer(POSITIONED);+ layer("mix-blend-mode: multiply", holdsBlend);+ const stacked = layer(`${POSITIONED}; z-index: 1`);+ layer("position: relative; z-index: 2", stacked);+ setPreviewRasterScale(0.5);+ expect([unpositioned, holdsFixed, holdsZIndex, holdsBlend].map(marked)).toEqual([+ false,+ false,+ false,+ false,+ ]);+ expect(marked(stacked)).toBe(true);+ });++ it("restores the document once shown at full size", () => {+ const words = layer(POSITIONED);+ const before = document.documentElement.outerHTML;+ setPreviewRasterScale(0.5);+ expect(document.documentElement.outerHTML).not.toBe(before);+ setPreviewRasterScale(1);+ expect(document.documentElement.outerHTML).toBe(before);+ expect(getComputedStyle(words).willChange).toBe("transform");+ });++ it("follows layers added and classes changed while the preview is small", async () => {+ const sheet = document.createElement("style");+ sheet.textContent = ".pinned { position: fixed }";+ document.head.append(sheet);+ setPreviewRasterScale(0.5);+ const late = layer(POSITIONED);+ const child = layer("", late);+ await Promise.resolve();+ expect(marked(late)).toBe(true);+ child.className = "pinned";+ await Promise.resolve();+ expect(marked(late)).toBe(false);+ sheet.remove();+ });++ it("gives the hint back when an inline style makes a child depend on it", async () => {+ const words = layer(POSITIONED);+ const word = layer("position: absolute", words);+ setPreviewRasterScale(0.5);+ expect(marked(words)).toBe(true);+ word.style.zIndex = "5";+ await Promise.resolve();+ expect(marked(words)).toBe(false);+ });++ it("re-reads a marked layer's own hints and position when they change", async () => {+ const sheet = document.createElement("style");+ sheet.textContent =+ ".hinted { will-change: transform } .hinted.fading { will-change: transform, opacity }";+ document.head.append(sheet);+ const classed = layer("position: absolute");+ classed.className = "hinted";+ const inline = layer(POSITIONED);+ const inlineHint = layer(POSITIONED);+ setPreviewRasterScale(0.5);+ expect([classed, inline, inlineHint].map(marked)).toEqual([true, true, true]);+ classed.className = "hinted fading";+ inline.style.position = "static";+ inlineHint.style.willChange = "transform, opacity";+ await Promise.resolve();+ expect([classed, inline, inlineHint].map(marked)).toEqual([false, false, false]);+ sheet.remove();+ });++ it("follows stylesheets and attribute selectors added while the preview is small", async () => {+ const byStylesheet = layer(POSITIONED);+ layer("", byStylesheet).className = "late-pinned";+ const byAttribute = layer(POSITIONED);+ const scene = layer("", byAttribute);+ setPreviewRasterScale(0.5);+ expect([byStylesheet, byAttribute].map(marked)).toEqual([true, true]);+ const sheet = document.createElement("style");+ sheet.textContent = '.late-pinned, [data-scene="pinned"] { position: fixed }';+ document.head.append(sheet);+ await Promise.resolve();+ expect(marked(byStylesheet)).toBe(false);+ expect(marked(byAttribute)).toBe(true);+ scene.dataset.scene = "pinned";+ await Promise.resolve();+ expect(marked(byAttribute)).toBe(false);+ sheet.remove();+ });++ it("leaves marked layers alone when only something around them changes", async () => {+ const scene = layer("");+ const words = [layer(POSITIONED, scene), layer(POSITIONED, scene)];+ setPreviewRasterScale(0.5);+ const markChanges: MutationRecord[] = [];+ const watch = new MutationObserver((records) => markChanges.push(...records));+ watch.observe(document.body, { subtree: true, attributeFilter: [PREVIEW_RASTER_ATTR] });+ scene.dataset.upcoming = "";+ await Promise.resolve();+ await Promise.resolve();+ watch.disconnect();+ expect(words.map(marked)).toEqual([true, true]);+ expect(markChanges).toEqual([]);+ });++ it("does not react to its own marks", async () => {+ const words = layer(POSITIONED);+ setPreviewRasterScale(0.5);+ words.dataset.scene = "intro";+ await Promise.resolve();+ await Promise.resolve();+ const reads = vi.spyOn(window, "getComputedStyle");+ await Promise.resolve();+ await Promise.resolve();+ expect(reads).not.toHaveBeenCalled();+ reads.mockRestore();+ expect(marked(words)).toBe(true);+ });++ it("re-checks the page when a linked stylesheet loads", async () => {+ const sheet = document.createElement("style");+ document.head.append(sheet);+ const words = layer(POSITIONED);+ layer("", words).className = "loaded-pinned";+ setPreviewRasterScale(0.5);+ const link = document.createElement("link");+ document.head.append(link);+ await Promise.resolve();+ expect(marked(words)).toBe(true);+ // Stands in for the linked rules arriving: stylesheet object edits are not observed.+ sheet.sheet!.insertRule(".loaded-pinned { position: fixed }");+ link.dispatchEvent(new Event("load"));+ expect(marked(words)).toBe(false);+ sheet.remove();+ link.remove();+ });+});diff --git a/packages/core/src/runtime/previewRasterHints.ts b/packages/core/src/runtime/previewRasterHints.tsnew file mode 100644index 0000000000..e31c3cdf26--- /dev/null+++ b/packages/core/src/runtime/previewRasterHints.ts@@ -0,0 +1,159 @@+import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";+import { isElementNode } from "./domRealm";++// Chromium rasters a `will-change: transform` layer at no less than device scale, so a preview shown+// small rasters each one at full size. Below 1x the hint is dropped wherever nothing depends on the+// stacking context and fixed-position containing block it makes, so the preview paints the same.+const TRANSFORM_HINTS = new Set(["transform", "translate", "rotate", "scale"]);+const MARKED = `[${PREVIEW_RASTER_ATTR}]`;+const DROP_HINTS = `${MARKED} { will-change: auto !important; }`;++let sheet: HTMLStyleElement | null = null;+let observer: MutationObserver | null = null;+const markedWith = new WeakMap<Element, string>();++// Other hints (opacity, filter) make boundaries of their own, so only a pure transform hint goes.+function onlyTransformHints(style: CSSStyleDeclaration): boolean {+ return style.willChange.split(",").every((hint) => TRANSFORM_HINTS.has(hint.trim()));+}++// Fixed boxes always need the hint's containing block; z-indexed and blending boxes need its+// stacking context unless the layer stacks anyway.+function escapes(element: Element, stacks: boolean, self = true): boolean {+ for (const pseudo of self ? [null, "::before", "::after"] : ["::before", "::after"]) {+ const style = getComputedStyle(element, pseudo);+ if (pseudo && style.content === "none") continue;+ if (style.position === "fixed") return true;+ if (!stacks && (style.zIndex !== "auto" || style.mixBlendMode !== "normal")) return true;+ }+ return false;+}++// A positioned element already contains its absolute descendants and paints in the same phase.+function canDropHint(element: Element, style: CSSStyleDeclaration): boolean {+ if (style.position === "static") return false;+ const stacks = style.zIndex !== "auto";+ if (escapes(element, stacks, false)) return false;+ for (const descendant of element.querySelectorAll("*")) {+ if (escapes(descendant, stacks)) return false;+ }+ return true;+}++// What a script can change inline on a marked layer that decides whether it stays marked.+function inlineFit(element: Element, style: CSSStyleDeclaration): string {+ return `${style.zIndex}|${(element as HTMLElement).style?.willChange ?? ""}`;+}++// A marked element reads `auto` through our rule. Lifting the mark to read its own hints repaints+// the layer, so that only happens when the element itself changed.+function fits(element: Element, readOwnHints: boolean): boolean {+ if (element === sheet) return false;+ const marked = element.hasAttribute(PREVIEW_RASTER_ATTR);+ if (marked && readOwnHints) element.removeAttribute(PREVIEW_RASTER_ATTR);+ const style = getComputedStyle(element);+ const hintsFit = (marked && !readOwnHints) || onlyTransformHints(style);+ const drop = hintsFit && canDropHint(element, style);+ if (drop) markedWith.set(element, inlineFit(element, style));+ return drop;+}++function check(element: Element, readOwnHints: boolean): void {+ const drop = fits(element, readOwnHints);+ if (drop !== element.hasAttribute(PREVIEW_RASTER_ATTR)) {+ element.toggleAttribute(PREVIEW_RASTER_ATTR, drop);+ }+}++function unmarkHolders(element: Element): void {+ let holder = element.parentElement?.closest(MARKED);+ while (holder) {+ if (escapes(element, getComputedStyle(holder).zIndex !== "auto")) {+ holder.removeAttribute(PREVIEW_RASTER_ATTR);+ }+ holder = holder.parentElement?.closest(MARKED);+ }+}++function followTree(root: Element): void {+ for (const element of [root, ...root.querySelectorAll("*")]) {+ check(element, element === root);+ unmarkHolders(element);+ }+}++// Scripts restyle inline every frame, so only what can break a mark is looked at.+function followStyle(element: Element): void {+ if (element.hasAttribute(PREVIEW_RASTER_ATTR)) {+ const style = getComputedStyle(element);+ if (style.position === "static" || inlineFit(element, style) !== markedWith.get(element)) {+ check(element, true);+ }+ }+ unmarkHolders(element);+}++function clearMarks(): void {+ for (const element of document.querySelectorAll(MARKED)) {+ element.removeAttribute(PREVIEW_RASTER_ATTR);+ }+}++// Every mark is decided before any is written, so the page restyles twice, not once per layer.+function markLayers(): void {+ clearMarks();+ const layers = [...document.querySelectorAll("*")].filter((element) => fits(element, true));+ for (const layer of layers) layer.setAttribute(PREVIEW_RASTER_ATTR, "");+}++function followAttribute(record: MutationRecord): void {+ if (!isElementNode(record.target) || record.attributeName === PREVIEW_RASTER_ATTR) return;+ // Any attribute can match a selector; inline style is the per-frame case, so it is cheaper.+ if (record.attributeName === "style") followStyle(record.target);+ else followTree(record.target);+}++// New rules can restyle anything, and a linked sheet only applies once it loads.+function addedRules(record: MutationRecord): Element[] {+ return [...record.addedNodes].filter(+ (node): node is Element =>+ isElementNode(node) && (node.localName === "style" || node.localName === "link"),+ );+}++function follow(records: MutationRecord[]): void {+ const rules = records.flatMap(addedRules);+ for (const rule of rules) {+ rule.addEventListener("load", () => observer && markLayers(), { once: true });+ }+ if (rules.length) return markLayers();+ for (const record of records) {+ if (record.type === "attributes") followAttribute(record);+ else for (const node of record.addedNodes) if (isElementNode(node)) followTree(node);+ }+}++function restore(): void {+ observer?.disconnect();+ observer = null;+ sheet?.remove();+ sheet = null;+ clearMarks();+}++/** The host reports how large it shows this document; below 1x, transform hints stop forcing full-size rasters. */+export function setPreviewRasterScale(scale: number): void {+ if (!(scale > 0) || !Number.isFinite(scale)) return;+ if (scale >= 1) return restore();+ if (observer) return;+ sheet = document.createElement("style");+ sheet.textContent = DROP_HINTS;+ (document.head ?? document.documentElement).append(sheet);+ markLayers();+ observer = new MutationObserver(follow);+ observer.observe(document.documentElement, {+ subtree: true,+ childList: true,+ attributes: true,+ });+}diff --git a/packages/core/src/runtime/stackingContext.test.ts b/packages/core/src/runtime/stackingContext.test.tsnew file mode 100644index 0000000000..6df58e250f--- /dev/null+++ b/packages/core/src/runtime/stackingContext.test.ts@@ -0,0 +1,20 @@+import { afterEach, describe, expect, it } from "vitest";+import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";+import { ROOT_CSS_STACKING_CONTEXT_ID, resolveCssStackingContextId } from "./stackingContext";++afterEach(() => {+ document.body.innerHTML = "";+});++describe("resolveCssStackingContextId", () => {+ it("keeps the stacking context of a layer whose hint the small preview dropped", () => {+ document.body.innerHTML = `+ <div id="layer" style="position: absolute; will-change: auto" ${PREVIEW_RASTER_ATTR}>+ <div id="clip"></div>+ </div>`;+ const clip = document.getElementById("clip")!;+ expect(resolveCssStackingContextId(clip)).not.toBe(ROOT_CSS_STACKING_CONTEXT_ID);+ document.getElementById("layer")!.removeAttribute(PREVIEW_RASTER_ATTR);+ expect(resolveCssStackingContextId(clip)).toBe(ROOT_CSS_STACKING_CONTEXT_ID);+ });+});diff --git a/packages/core/src/runtime/stackingContext.ts b/packages/core/src/runtime/stackingContext.tsindex e27f77c751..9aa226bb59 100644--- a/packages/core/src/runtime/stackingContext.ts+++ b/packages/core/src/runtime/stackingContext.ts@@ -1,3 +1,5 @@+import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";+export const ROOT_CSS_STACKING_CONTEXT_ID = "css:root";const NON_NONE_STACKING_CONTEXT_PROPERTIES = [@@ -67,6 +69,7 @@ function createsWillChangeStackingContext(style: CSSStyleDeclaration): boolean {}function createsCssStackingContext(element: HTMLElement, style: CSSStyleDeclaration): boolean {+ if (element.hasAttribute(PREVIEW_RASTER_ATTR)) return true;if (createsPositionedStackingContext(style)) return true;if (createsZIndexedStackingContext(element, style)) return true;if (createsVisualEffectStackingContext(style)) return true;diff --git a/packages/core/src/studioPreviewMark.ts b/packages/core/src/studioPreviewMark.tsindex dabeb6ba1b..58dec41377 100644--- a/packages/core/src/studioPreviewMark.ts+++ b/packages/core/src/studioPreviewMark.ts@@ -2,3 +2,9 @@ export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming";export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy";export const STUDIO_PREVIEW_ERRORS = "__hfPreviewErrors";+export const PREVIEW_RASTER_ATTR = "data-hf-preview-raster";+export const PREVIEW_ONLY_ATTRS = [+ STUDIO_PREVIEW_UPCOMING_ATTR,+ STUDIO_PREVIEW_LAZY_ATTR,+ PREVIEW_RASTER_ATTR,+] as const;diff --git a/packages/player/package.json b/packages/player/package.jsonindex 2cc8bba89c..ea8503547c 100644--- a/packages/player/package.json+++ b/packages/player/package.json@@ -32,6 +32,7 @@"typecheck": "tsc --noEmit && tsc --noEmit -p tests/perf/tsconfig.json","test": "vitest run","test:browser-security": "bun run tests/browser/sandbox-origin.ts",+ "test:browser-raster": "bun run tests/browser/preview-raster-size.ts","perf": "bun run tests/perf/index.ts"},"dependencies": {diff --git a/packages/player/tests/browser/preview-raster-size.ts b/packages/player/tests/browser/preview-raster-size.tsnew file mode 100644index 0000000000..0b2cc8e6b0--- /dev/null+++ b/packages/player/tests/browser/preview-raster-size.ts@@ -0,0 +1,256 @@+import assert from "node:assert/strict";++import { launchBrowser } from "../perf/runner.js";+import { startServer } from "../perf/server.js";++// A small preview must raster its composition's will-change layers near the size it shows them, not at+// full size (which runs heavy compositions out of tile memory), without moving or repainting anything.+const server = startServer({ noCache: true });+const browser = await launchBrowser({ width: 1920, height: 1080 });++type Layer = {+ ideal_contents_scale?: number;+ raster_scales?: { contents_scale?: [number, number] };+};++try {+ const page = await browser.newPage();+ await page.evaluateOnNewDocument(() => {+ const sheet = new CSSStyleSheet();+ sheet.replaceSync("hyperframes-player { display: block; width: 1920px; height: 1080px }");+ document.adoptedStyleSheets = [sheet];+ });+ await page.goto(`${server.origin}/host.html?fixture=preview-raster`);+ await page.waitForFunction(() => window.__playerReady === true);+ const frame = page.frames().find((candidate) => candidate.url().includes("/fixtures/"));+ assert(frame, "composition frame loaded");+ const until = (what: string, check: () => boolean) =>+ frame+ .waitForFunction(check, { timeout: 10_000 })+ .catch(() => assert.fail(`timed out: ${what}`));+ const swapped = () => getComputedStyle(document.querySelector(".lyric")!).willChange === "auto";+ const hinted = () =>+ getComputedStyle(document.querySelector(".lyric")!).willChange === "transform";+ const resize = (width: number, height: number) =>+ page.evaluate(+ (w, h) =>+ Object.assign(document.getElementById("player")!.style, {+ width: `${w}px`,+ height: `${h}px`,+ }),+ width,+ height,+ );+ // What the player sends; lets the test turn the swap off and on at one shown size.+ const sendDisplayScale = (scale: number) =>+ frame.evaluate(+ (value) =>+ window.postMessage(+ { source: "hf-parent", type: "control", action: "set-display-scale", scale: value },+ "*",+ ),+ scale,+ );+ const settle = () =>+ page.evaluate(+ () => new Promise((done) => requestAnimationFrame(() => requestAnimationFrame(done))),+ );+ const seek = async (time: number) => {+ await page.evaluate(+ (t) => (document.getElementById("player") as HTMLElement & { seek(t: number): void }).seek(t),+ time,+ );+ await settle();+ };+ // Every box, what paints at the stacking and 3D probes, every inline style and computed filter.+ const snapshot = () =>+ frame.evaluate(() => {+ const elements = [...document.body.querySelectorAll("*")];+ return {+ boxes: elements.map((element) => {+ const box = element.getBoundingClientRect();+ return [box.x, box.y, box.width, box.height];+ }),+ stacking: document.elementFromPoint(1000, 800)?.id,+ flipped: document.elementFromPoint(1500, 800)?.id,+ styles: elements.map((element) => (element as HTMLElement).style?.cssText ?? ""),+ filters: elements.map((element) => getComputedStyle(element).filter),+ };+ });+ // A screenshot of a composition element's interior, 2px in from its antialiased edges.+ const shot = async (id: string) => {+ const [x, y, width, height] = await frame.evaluate((target) => {+ const box = document.getElementById(target)!.getBoundingClientRect();+ return [box.x, box.y, box.width, box.height];+ }, id);+ const [left, top, scale] = await page.evaluate(() => {+ const box = document.getElementById("player")!.getBoundingClientRect();+ return [box.x, box.y, box.width / 1920];+ });+ const clip = {+ x: left + x * scale + 2,+ y: top + y * scale + 2,+ width: width * scale - 4,+ height: height * scale - 4,+ };+ return page.screenshot({ encoding: "base64", clip });+ };+ const changedShare = (a: string, b: string) =>+ page.evaluate(+ async (first, second) => {+ const pixels = async (png: string) => {+ const image = new Image();+ image.src = `data:image/png;base64,${png}`;+ await image.decode();+ const canvas = new OffscreenCanvas(image.width, image.height);+ const context = canvas.getContext("2d")!;+ context.drawImage(image, 0, 0);+ return context.getImageData(0, 0, image.width, image.height).data;+ };+ const [p, q] = [await pixels(first), await pixels(second)];+ let changed = 0;+ for (let i = 0; i < p.length; i += 4) {+ const delta = [0, 1, 2].map((c) => Math.abs(p[i + c]! - q[i + c]!));+ if (Math.max(...delta) > 32) changed++;+ }+ return changed / (p.length / 4);+ },+ a,+ b,+ );+ // A layer snapshot is only written on a compositor draw, so each trace seeks to force one.+ const maxOversize = async (attempt: number) => {+ await page.tracing.start({ categories: ["disabled-by-default-cc.debug"] });+ await seek(attempt % 6);+ const trace = JSON.parse(new TextDecoder().decode(await page.tracing.stop())) as {+ traceEvents: { args?: { snapshot?: { active_tree?: { layers?: Layer[] } } } }[];+ };+ const ratios = trace.traceEvents+ .flatMap((event) => event.args?.snapshot?.active_tree?.layers ?? [])+ .filter((layer) => layer.raster_scales?.contents_scale && layer.ideal_contents_scale)+ .map((layer) => layer.raster_scales!.contents_scale![0] / layer.ideal_contents_scale!);+ return Math.max(0, ...ratios);+ };++ const full = await snapshot();+ assert.equal(await frame.evaluate(hinted), true, "shown at full size, the hints stay");+ assert.equal(full.stacking, "cover", "the hinted stack keeps its z-index child under a sibling");+ assert.equal(full.flipped, "card", "the flipped card hides its front");++ await resize(528, 297);+ await until("a small preview swaps the hints", swapped);+ const small = await snapshot();+ small.boxes.forEach((box, index) =>+ assert(+ box.every((value, edge) => Math.abs(value - full.boxes[index]![edge]!) < 0.01),+ `element ${index} moved from ${full.boxes[index]} to ${box}`,+ ),+ );+ assert.equal(small.stacking, "cover", "paint order is unchanged");+ assert.equal(small.flipped, "card", "a preserve-3d container is not flattened");+ assert.deepEqual(small.filters, full.filters, "no computed filter changes");+ assert.deepEqual(small.styles, full.styles, "no inline style is written");+ assert.deepEqual(+ await frame.evaluate(() =>+ ["wrap", "stack", "card", "glow", "mixed", "glass-wrap", "tilt-wrap"].map(+ (id) => getComputedStyle(document.getElementById(id)!).willChange,+ ),+ ),+ ["transform", "transform", "transform", "transform", "transform, opacity", "auto", "auto"],+ "only layers nothing depends on lose the hint",+ );+ await frame.evaluate(() => {+ const late = document.createElement("div");+ late.className = "lyric";+ late.id = "late";+ document.getElementById("lyrics")!.append(late);+ });+ await until(+ "a layer added later is swapped",+ () => getComputedStyle(document.getElementById("late")!).willChange === "auto",+ );+ await frame.evaluate(() => document.getElementById("late")!.remove());++ // The same small preview with the swap off and on: pixels match, and only the swap rasters small.+ // A capture can land before re-rastered tiles do, so each shot waits until two in a row agree.+ const steadyShot = async (id: string) => {+ let last = await shot(id);+ for (let attempt = 0; attempt < 20; attempt++) {+ const next = await shot(id);+ if (next === last) return next;+ last = next;+ }+ return assert.fail(`#${id} never stopped changing`);+ };+ const shots = async () => {+ await seek(0);+ return [await steadyShot("glass"), await steadyShot("tilt-wrap")];+ };+ // Only the swap may differ between the two halves; layers that keep their hint on purpose go.+ await frame.evaluate(() => {+ for (const id of ["wrap", "stack", "stage3d", "glow", "mixed"]) {+ document.getElementById(id)!.remove();+ }+ });+ const swappedShots = await shots();+ // Toggling the swap restyles the page and the player answers with its real scale, so its own+ // sends are held while the test decides the scale.+ await page.evaluate(() => {+ const player = document.getElementById("player") as HTMLElement & { _sendDisplayScale(): void };+ player._sendDisplayScale = () => {};+ });+ await sendDisplayScale(1);+ await until("the swap turns off", hinted);+ let unswapped = 0;+ for (let attempt = 0; attempt < 20 && unswapped <= 1.5; attempt++) {+ unswapped = await maxOversize(attempt);+ }+ assert.ok(unswapped > 1.5, `without the swap the trace shows ${unswapped}x, not oversized`);+ const nativeShots = await shots();+ assert.deepEqual(+ await frame.evaluate(() =>+ ["glass-wrap", "tilt-wrap"].map(+ (id) => getComputedStyle(document.getElementById(id)!).willChange,+ ),+ ),+ ["transform", "transform"],+ "the native shots are taken with the swap off",+ );+ const glass = await changedShare(swappedShots[0]!, nativeShots[0]!);+ assert.ok(glass <= 0.1, `frosted glass changed in ${(glass * 100).toFixed(0)}% of its pixels`);+ const tilt = await changedShare(swappedShots[1]!, nativeShots[1]!);+ assert.ok(tilt <= 0.1, `a 3D child of a swapped layer changed in ${(tilt * 100).toFixed(0)}%`);++ await sendDisplayScale(528 / 1920);+ await until("the swap turns back on", swapped);+ for (let clean = 0, attempt = 0; clean < 3; attempt++) {+ const oversize = await maxOversize(attempt);+ clean = oversize <= 1.5 ? clean + 1 : 0;+ assert.ok(attempt < 20, `a layer is still rastered at ${oversize.toFixed(2)}x the size shown`);+ }++ // Scripts restyle inline: a child given a z-index gives its layer the hint back.+ await frame.evaluate(() => {+ document.getElementById("tilted")!.style.zIndex = "5";+ });+ await until(+ "an inline z-index gives the hint back",+ () => getComputedStyle(document.getElementById("tilt-wrap")!).willChange === "transform",+ );++ await page.evaluate(() => {+ delete (document.getElementById("player") as HTMLElement & { _sendDisplayScale?: unknown })+ ._sendDisplayScale;+ });+ await resize(1920, 1080);+ await until("full size restores the hints", hinted);+ assert.equal(+ await frame.evaluate(() => document.querySelectorAll("[data-hf-preview-raster]").length),+ 0,+ "full size removes every mark",+ );+ console.log("preview rasters at its shown size with layout and pixels unchanged: PASS");+} finally {+ await browser.close();+ await server.stop();+}diff --git a/packages/player/tests/perf/fixtures/preview-raster/index.html b/packages/player/tests/perf/fixtures/preview-raster/index.htmlnew file mode 100644index 0000000000..8cfdadb07c--- /dev/null+++ b/packages/player/tests/perf/fixtures/preview-raster/index.html@@ -0,0 +1,204 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="utf-8" />+ <style>+ html,+ body {+ margin: 0;+ width: 1920px;+ height: 1080px;+ overflow: hidden;+ background: #111;+ font: 600 64px sans-serif;+ color: #fff;+ }+ #root {+ position: relative;+ width: 1920px;+ height: 1080px;+ }+ .lyric {+ position: absolute;+ will-change: transform;+ }+ #wrap {+ position: absolute;+ left: 200px;+ top: 700px;+ width: 400px;+ height: 200px;+ will-change: transform;+ }+ #fixed {+ position: fixed;+ left: 10px;+ top: 10px;+ width: 50px;+ height: 50px;+ background: #f80;+ }+ #stack {+ position: absolute;+ left: 900px;+ top: 700px;+ width: 200px;+ height: 200px;+ will-change: transform;+ }+ #lifted {+ position: absolute;+ inset: 0;+ z-index: 5;+ background: #0af;+ }+ #cover {+ position: absolute;+ left: 950px;+ top: 750px;+ width: 100px;+ height: 100px;+ z-index: 1;+ background: #a0f;+ }+ #stage3d {+ position: absolute;+ left: 1400px;+ top: 700px;+ width: 200px;+ height: 200px;+ perspective: 600px;+ }+ #card {+ width: 200px;+ height: 200px;+ will-change: transform;+ transform-style: preserve-3d;+ transform: rotateY(180deg);+ }+ #front {+ position: absolute;+ inset: 0;+ backface-visibility: hidden;+ background: #f00;+ }+ #blurred {+ position: absolute;+ left: 1700px;+ top: 100px;+ width: 100px;+ height: 100px;+ will-change: transform;+ filter: blur(2px);+ background: #0f0;+ }+ #stripes {+ position: absolute;+ left: 1660px;+ top: 570px;+ width: 240px;+ height: 120px;+ background: repeating-linear-gradient(90deg, #fff 0 12px, #000 12px 24px);+ }+ #glass-wrap {+ position: absolute;+ left: 1680px;+ top: 590px;+ width: 200px;+ height: 80px;+ will-change: transform;+ }+ #glass {+ position: absolute;+ inset: 0;+ backdrop-filter: blur(8px);+ }+ #tilt-wrap {+ position: absolute;+ left: 640px;+ top: 930px;+ width: 200px;+ height: 120px;+ will-change: transform;+ }+ #tilted {+ width: 200px;+ height: 120px;+ transform: rotateY(50deg);+ background: #fc0;+ }+ #glow {+ position: absolute;+ left: 1300px;+ top: 950px;+ width: 120px;+ height: 80px;+ will-change: transform;+ background: #333;+ }+ #glow::before {+ content: "";+ position: absolute;+ inset: -10px;+ z-index: -1;+ background: #f0f;+ }+ #mixed {+ position: absolute;+ left: 1500px;+ top: 950px;+ width: 120px;+ height: 80px;+ will-change: transform, opacity;+ background: #444;+ }+ </style>+ <script data-hyperframes-runtime="1" src="/vendor/hyperframe.runtime.iife.js"></script>+ <script src="/vendor/gsap.min.js"></script>+ </head>+ <body>+ <div+ id="root"+ data-composition-id="main"+ data-width="1920"+ data-height="1080"+ data-duration="6"+ >+ <div id="lyrics"></div>+ <div id="wrap"><div id="fixed"></div></div>+ <div id="stack"><div id="lifted"></div></div>+ <div id="cover"></div>+ <div id="stage3d">+ <div id="card"><div id="front"></div></div>+ </div>+ <div id="blurred"></div>+ <div id="stripes"></div>+ <div id="glass-wrap"><div id="glass"></div></div>+ <div id="tilt-wrap"><div id="tilted"></div></div>+ <div id="glow"></div>+ <div id="mixed"></div>+ <div style="display: none"><h2 class="lyric">Hidden</h2></div>+ </div>+ <script>+ const words = ["HEY", "LOUD", "OVER", "WALL", "SAY", "YEAH", "NOW", "AGAIN"];+ const lyrics = document.getElementById("lyrics");+ const tl = gsap.timeline({ paused: true });+ for (let i = 0; i < 24; i++) {+ const word = document.createElement("div");+ word.className = "lyric";+ word.textContent = words[i % words.length];+ word.style.left = `${80 + (i % 6) * 280}px`;+ word.style.top = `${60 + Math.floor(i / 6) * 140}px`;+ lyrics.append(word);+ tl.fromTo(+ word,+ { x: -40, scale: 0.9 },+ { x: 40, scale: 1.1, duration: 6, ease: "none" },+ 0,+ );+ }+ window.__timelines = window.__timelines || {};+ window.__timelines.main = tl;+ </script>+ </body>+</html>diff --git a/packages/studio/src/components/editor/domEditingLayers.ts b/packages/studio/src/components/editor/domEditingLayers.tsindex 287c0a1e68..be37659066 100644--- a/packages/studio/src/components/editor/domEditingLayers.ts+++ b/packages/studio/src/components/editor/domEditingLayers.ts@@ -1,3 +1,4 @@+import { PREVIEW_ONLY_ATTRS } from "@hyperframes/core/studio-preview-mark";import { knownSourceAnswer, probeSourceElement, type ProbeTarget } from "./probeSourceElement";import { isAudibleVideoNode } from "../../player/lib/timelineElementHelpers";import type { PatchOperation } from "../../utils/sourcePatcher";@@ -55,6 +56,8 @@ function getTextFieldLabel(return `Text ${index + 1}`;}+const NOT_COPIED_ATTRS = new Set<string>(["style", ...PREVIEW_ONLY_ATTRS]);+function buildTextField(el: HTMLElement,index: number,@@ -70,7 +73,7 @@ function buildTextField(value: el.textContent ?? "",tagName,attributes: Array.from(el.attributes)- .filter((attribute) => attribute.name !== "style")+ .filter((attribute) => !NOT_COPIED_ATTRS.has(attribute.name)).map((attribute) => ({name: attribute.name,value: attribute.value,diff --git a/packages/studio/src/components/editor/domEditingTextFields.test.ts b/packages/studio/src/components/editor/domEditingTextFields.test.tsindex 5c3e7bd23a..e0732bf390 100644--- a/packages/studio/src/components/editor/domEditingTextFields.test.ts+++ b/packages/studio/src/components/editor/domEditingTextFields.test.ts@@ -1,5 +1,11 @@import { describe, expect, it } from "vitest";+import { Window } from "happy-dom";+import {+ PREVIEW_RASTER_ATTR,+ STUDIO_PREVIEW_UPCOMING_ATTR,+} from "@hyperframes/core/studio-preview-mark";import { serializeDomEditTextFields } from "./domEditing";+import { collectDomEditTextFields } from "./domEditingLayers";describe("serializeDomEditTextFields — mixed content", () => {it("round-trips text-node + child element fields", () => {@@ -58,3 +64,15 @@ describe("serializeDomEditTextFields — mixed content", () => {).toBe("A < B & C > D");});});++describe("collectDomEditTextFields — preview marks", () => {+ it("keeps preview-only attributes out of a saved text edit", () => {+ const { document } = new Window();+ document.body.innerHTML = `<h1>Hello <span class="word" ${PREVIEW_RASTER_ATTR} ${STUDIO_PREVIEW_UPCOMING_ATTR}>world</span></h1>`;+ const heading = document.querySelector("h1") as unknown as HTMLElement;+ const saved = serializeDomEditTextFields(collectDomEditTextFields(heading));+ expect(saved).toContain('<span class="word" data-hf-text-key=');+ expect(saved).not.toContain(PREVIEW_RASTER_ATTR);+ expect(saved).not.toContain(STUDIO_PREVIEW_UPCOMING_ATTR);+ });+});diff --git a/packages/studio/src/utils/authoredSource.test.ts b/packages/studio/src/utils/authoredSource.test.tsnew file mode 100644index 0000000000..e4e6d26a52--- /dev/null+++ b/packages/studio/src/utils/authoredSource.test.ts@@ -0,0 +1,25 @@+// @vitest-environment happy-dom+import { describe, expect, it } from "vitest";+import {+ PREVIEW_RASTER_ATTR,+ STUDIO_PREVIEW_LAZY_ATTR,+} from "@hyperframes/core/studio-preview-mark";+import { liveMarkupWithoutPreviewMarks } from "./authoredSource";++describe("liveMarkupWithoutPreviewMarks", () => {+ it("drops the preview's raster mark from an element and its descendants", () => {+ const live = document.createElement("div");+ live.innerHTML = `<div class="line" ${PREVIEW_RASTER_ATTR}><span class="word">LOUD</span></div>`;+ live.setAttribute(PREVIEW_RASTER_ATTR, "");+ expect(liveMarkupWithoutPreviewMarks(live)).toBe(+ '<div><div class="line"><span class="word">LOUD</span></div></div>',+ );+ expect(live.hasAttribute(PREVIEW_RASTER_ATTR)).toBe(true);+ });++ it("gives a lazily previewed image back its authored loading", () => {+ const live = document.createElement("div");+ live.innerHTML = `<img src="a.png" loading="lazy" ${STUDIO_PREVIEW_LAZY_ATTR}>`;+ expect(liveMarkupWithoutPreviewMarks(live)).toBe('<div><img src="a.png"></div>');+ });+});diff --git a/packages/studio/src/utils/authoredSource.ts b/packages/studio/src/utils/authoredSource.tsindex 70d699d898..8068d816bb 100644--- a/packages/studio/src/utils/authoredSource.ts+++ b/packages/studio/src/utils/authoredSource.ts@@ -1,7 +1,7 @@import { ensureHfIds, isCompositionTemplate } from "@hyperframes/parsers/hf-ids";import {+ PREVIEW_ONLY_ATTRS,STUDIO_PREVIEW_LAZY_ATTR,- STUDIO_PREVIEW_UPCOMING_ATTR,} from "@hyperframes/core/studio-preview-mark";// Stamped as the preview stamps the files it serves, so a live element's hf-id finds its source.@@ -98,10 +98,8 @@ export function authoredMarkup(authored: Element, live: Element, sourceFile: strexport function liveMarkupWithoutPreviewMarks(live: Element): string {const copy = live.cloneNode(true) as Element;for (const el of [copy, ...Array.from(copy.querySelectorAll("*"))]) {- el.removeAttribute(STUDIO_PREVIEW_UPCOMING_ATTR);- if (!el.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) continue;- el.removeAttribute("loading");- el.removeAttribute(STUDIO_PREVIEW_LAZY_ATTR);+ if (el.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) el.removeAttribute("loading");+ for (const attr of PREVIEW_ONLY_ATTRS) el.removeAttribute(attr);}return copy.outerHTML;}