Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(studio): preview decodes hard-to-play videos at the size they are shown (#5184)
Miguel Ángelb66e8a195e5a
diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex e7a2e2fbc1..717d38f98c 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -264,6 +264,12 @@ export {parseFirstFrameColour,type ToneMapSourceColour,} from "./hdrToneMap";+export {+ PREVIEW_PROXY_BOX_PARAM,+ formatPreviewProxyBox,+ parsePreviewProxyBox,+ type PreviewProxyBox,+} from "./previewProxyBox";// Inline scriptsexport {diff --git a/packages/core/src/previewProxyBox.test.ts b/packages/core/src/previewProxyBox.test.tsnew file mode 100644index 0000000000..6d93e03c3f--- /dev/null+++ b/packages/core/src/previewProxyBox.test.ts@@ -0,0 +1,21 @@+import { describe, expect, it } from "vitest";+import { parsePreviewProxyBox, quantizePreviewProxyBox } from "./previewProxyBox";++describe("preview proxy box", () => {+ it("rounds each side up to the next rung", () => {+ expect(quantizePreviewProxyBox(1080, 1920)).toEqual({ width: 1448, height: 2048 });+ expect(quantizePreviewProxyBox(1, 256)).toEqual({ width: 256, height: 256 });+ });++ it("has no bound for an empty box or one past the largest rung", () => {+ expect(quantizePreviewProxyBox(0, 1920)).toBeNull();+ expect(quantizePreviewProxyBox(8193, 100)).toBeNull();+ });++ it("reads back only rungs", () => {+ expect(parsePreviewProxyBox("1448x2048")).toEqual({ width: 1448, height: 2048 });+ expect(parsePreviewProxyBox("1449x2048")).toBeNull();+ expect(parsePreviewProxyBox("1448x2048x1")).toBeNull();+ expect(parsePreviewProxyBox("")).toBeNull();+ });+});diff --git a/packages/core/src/previewProxyBox.ts b/packages/core/src/previewProxyBox.tsnew file mode 100644index 0000000000..ae848ded52--- /dev/null+++ b/packages/core/src/previewProxyBox.ts@@ -0,0 +1,32 @@+/** Query parameter carrying the largest box, in device pixels, a preview shows a proxied video at. */+export const PREVIEW_PROXY_BOX_PARAM = "hf-proxy-box";++export type PreviewProxyBox = { width: number; height: number };++// 256 * 2^(k/2), even: nearby boxes share one cached copy, and a rung overshoots by at most ~41%.+const RUNGS = [256, 362, 512, 724, 1024, 1448, 2048, 2896, 4096, 5792, 8192];++function rungAtLeast(pixels: number): number | null {+ if (!(pixels > 0)) return null;+ return RUNGS.find((rung) => rung >= pixels) ?? null;+}++/** Rounds a shown box up to the size ladder; null when it is empty or larger than any rung. */+export function quantizePreviewProxyBox(width: number, height: number): PreviewProxyBox | null {+ const w = rungAtLeast(width);+ const h = rungAtLeast(height);+ return w && h ? { width: w, height: h } : null;+}++export function formatPreviewProxyBox(box: PreviewProxyBox): string {+ return `${box.width}x${box.height}`;+}++/** Accepts only ladder rungs, so a client cannot make one cached copy per pixel. */+export function parsePreviewProxyBox(raw: string): PreviewProxyBox | null {+ const match = /^(\d{3,4})x(\d{3,4})$/.exec(raw);+ if (!match) return null;+ const width = Number(match[1]);+ const height = Number(match[2]);+ return RUNGS.includes(width) && RUNGS.includes(height) ? { width, height } : null;+}diff --git a/packages/core/src/runtime/bridge.ts b/packages/core/src/runtime/bridge.tsindex 919836e001..a5a5e89bbd 100644--- a/packages/core/src/runtime/bridge.ts+++ b/packages/core/src/runtime/bridge.ts@@ -32,6 +32,7 @@ type BridgeDeps = {onDisablePickMode: () => void;onSetRuntimeData?: (channel: string, payload: unknown, requestId?: number) => void;onClearRuntimeData?: (channel: string, requestId?: number) => void;+ onSetDisplayScale?: (scale: number) => void;getCanonicalFps: () => number;};@@ -74,6 +75,7 @@ const CONTROL_HANDLERS = new Map<string, ControlHandler>(deps.onSetWebAudioMediaDisabled(Boolean(data.disabled)),"set-playback-rate": (data, deps) => deps.onSetPlaybackRate(Number(data.playbackRate ?? 1)),"set-idle-heartbeat": (data, deps) => deps.onSetIdleHeartbeat(Boolean(data.slow)),+ "set-display-scale": (data, deps) => deps.onSetDisplayScale?.(Number(data.scale)),"set-root-duration": (data, deps) => deps.onSetRootDuration(Number(data.durationSeconds ?? 0)),"set-play-range": (data, deps) =>deps.onSetPlayRange(diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex ca56317ad3..9e39df6d68 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -44,7 +44,12 @@ import {syncRuntimeMedia,type RuntimeMediaClip,} from "./media";-import { handleErrorForProxy, handleMetadataForProxy, maybeProxyProactively } from "./mediaProxy";+import {+ handleErrorForProxy,+ handleMetadataForProxy,+ maybeProxyProactively,+ setProxyDisplayScale,+} from "./mediaProxy";import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";import { createPickerModule } from "./picker";import { createRuntimePlayer, resolveRenderSeekTime, type RuntimePlayerTransport } from "./player";@@ -5019,6 +5024,7 @@ export function initSandboxRuntimeModular(): void {if (state.transportClock) state.transportClock.setRate(state.playbackRate);applyWebAudioRate();},+ onSetDisplayScale: setProxyDisplayScale,onSetIdleHeartbeat: (slow) => {slowIdleHeartbeat = slow;wakeTransport();diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.tsindex 77fa32b20b..d3b385882e 100644--- a/packages/core/src/runtime/mediaProxy.test.ts+++ b/packages/core/src/runtime/mediaProxy.test.ts@@ -764,3 +764,108 @@ describe("deriveCodecMapKey", () => {expect(deriveCodecMapKey(el)).toBeNull();});});++describe("a copy sized to how large the video shows", () => {+ // A framed document learns its on-screen scale from its host; load a module that is framed.+ async function framedVideo(width: number, height: number) {+ vi.resetModules();+ vi.spyOn(window, "parent", "get").mockReturnValue({} as Window);+ const { maybeProxyProactively, setProxyDisplayScale } = await import("./mediaProxy");+ window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": HEVC_ENTRY };+ const el = createVideo("/video.mp4");+ stubCanPlayType(el, "");+ el.getBoundingClientRect = () => ({ width, height }) as DOMRect;+ Object.defineProperty(document.documentElement, "clientWidth", {+ configurable: true,+ value: 1080,+ });+ Object.defineProperty(document.documentElement, "clientHeight", {+ configurable: true,+ value: 1920,+ });+ vi.spyOn(window, "devicePixelRatio", "get").mockReturnValue(2);+ maybeProxyProactively(el);+ return { el, setProxyDisplayScale };+ }++ const boxOf = (el: HTMLMediaElement) =>+ new URL(el.src, document.baseURI).searchParams.get("hf-proxy-box");++ it("asks only once the host says how large it shows the frame, for that size", async () => {+ const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);+ await settle();+ expect(fetch).not.toHaveBeenCalled();++ // A 1080x1920 stage at half size on a 2x screen is 1080x1920 device pixels.+ setProxyDisplayScale(0.5);+ await settle();+ expect(boxOf(el)).toBe("1448x2048");+ expect(vi.mocked(fetch).mock.calls[0]?.[0]).toBe(el.src);+ });++ it("measures a video scaled past the stage by its own box", async () => {+ const { el, setProxyDisplayScale } = await framedVideo(2160, 3840);+ setProxyDisplayScale(0.5);+ await settle();+ expect(boxOf(el)).toBe("2896x4096");+ });++ it("asks for the source-size copy when the host never says", async () => {+ vi.useFakeTimers();+ const { el } = await framedVideo(1080, 1920);+ await vi.advanceTimersByTimeAsync(999);+ expect(fetch).not.toHaveBeenCalled();+ await vi.advanceTimersByTimeAsync(1);+ expect(isProxied(el)).toBe(true);+ expect(boxOf(el)).toBeNull();+ });++ it("moves to a larger copy when the video shows larger, and keeps it when it shrinks", async () => {+ const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);+ setProxyDisplayScale(0.5);+ await settle();++ setProxyDisplayScale(1);+ setProxyDisplayScale(1);+ await settle();+ expect(boxOf(el)).toBe("2896x4096");+ expect(el.load).toHaveBeenCalledTimes(2);++ setProxyDisplayScale(0.25);+ await settle();+ expect(boxOf(el)).toBe("2896x4096");+ expect(el.load).toHaveBeenCalledTimes(2);+ });++ it("loads the newest size asked for, even when an older, smaller copy is served first", async () => {+ const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);+ setProxyDisplayScale(0.5);+ await settle();++ setProxyDisplayScale(1);+ setProxyDisplayScale(2);+ await settle();+ expect(boxOf(el)).toBe("5792x8192");+ expect(el.load).toHaveBeenCalledTimes(2);+ });++ it("grows a first copy that was made while the frame grew", async () => {+ vi.useFakeTimers();+ let made = false;+ vi.stubGlobal(+ "fetch",+ vi.fn(async () =>+ made+ ? new Response("x", { status: 206 })+ : new Response(null, { status: 202, headers: { "Retry-After": "2" } }),+ ),+ );+ const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);+ setProxyDisplayScale(0.5);+ await vi.advanceTimersByTimeAsync(0);+ setProxyDisplayScale(1);+ made = true;+ await vi.advanceTimersByTimeAsync(2_000);+ expect(boxOf(el)).toBe("2896x4096");+ });+});diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.tsindex b7acd27022..eea5fa1c1e 100644--- a/packages/core/src/runtime/mediaProxy.ts+++ b/packages/core/src/runtime/mediaProxy.ts@@ -7,6 +7,12 @@ import { isVideoElement } from "./domRealm";import { swappedElements } from "./proxySrc";import { waitForServedProxy } from "./proxyWait";import { registerSeekCompletion } from "./adapters/seek-dispatch";+import {+ formatPreviewProxyBox,+ PREVIEW_PROXY_BOX_PARAM,+ quantizePreviewProxyBox,+ type PreviewProxyBox,+} from "../previewProxyBox";/*** One entry per project-root-relative asset pathname, injected by the@@ -57,6 +63,89 @@ function currentSrcValue(el: HTMLMediaElement): string {return el.currentSrc || el.src;}+// How large the host shows this document, in its CSS pixels per ours (player fit x page zoom).+// A top-level page shows itself at 1; a framed one waits for its host to say.+let displayScale: number | null = window.parent === window ? 1 : null;+let displayScaleArrived: () => void = () => {};+const displayScaleKnown = new Promise<void>((arrived) => (displayScaleArrived = arrived));+// A host that never says (an older player) gets the source-size copy after this wait.+const DISPLAY_SCALE_WAIT_MS = 1000;++// Elements on a size-bounded copy: the source, the copy's URL and the box it was made for.+type BoxedCopy = { originalSrc: string; variant: string; href: string; box: PreviewProxyBox };+const boxedElements = new Map<HTMLMediaElement, BoxedCopy>();+// The larger copy each element waits for; a newer ask replaces it and the older wait stops.+const pendingUpgrades = new WeakMap<HTMLMediaElement, string>();++function shownScale(): Promise<number | null> {+ if (displayScale !== null) return Promise.resolve(displayScale);+ return Promise.race([+ displayScaleKnown.then(() => displayScale),+ new Promise<null>((give) => setTimeout(() => give(null), DISPLAY_SCALE_WAIT_MS)),+ ]);+}++/** The video's box in device pixels, at least the stage so a later zoom or a clip still hidden fits. */+function shownBox(el: HTMLMediaElement, scale: number): PreviewProxyBox | null {+ // object-fit: none shows the source at its own size.+ if (getComputedStyle(el).objectFit === "none") return null;+ const rect = el.getBoundingClientRect();+ const stage = document.documentElement;+ const pixels = scale * (window.devicePixelRatio || 1);+ return quantizePreviewProxyBox(+ Math.max(rect.width, stage.clientWidth) * pixels,+ Math.max(rect.height, stage.clientHeight) * pixels,+ );+}++/** The host reports how large it shows this document; copies too small for it are replaced. */+export function setProxyDisplayScale(scale: number): void {+ if (!(scale > 0) || !Number.isFinite(scale)) return;+ displayScale = scale;+ displayScaleArrived();+ for (const el of boxedElements.keys()) keepSized(el, scale);+}++function keepSized(el: HTMLMediaElement, scale: number): void {+ const copy = boxedElements.get(el);+ if (!copy) return;+ if (!el.isConnected || el.getAttribute("src") !== copy.href) {+ boxedElements.delete(el);+ return;+ }+ const needed = shownBox(el, scale);+ if (needed && needed.width <= copy.box.width && needed.height <= copy.box.height) return;+ // Never smaller on either side than the copy it replaces.+ const box = needed && {+ width: Math.max(needed.width, copy.box.width),+ height: Math.max(needed.height, copy.box.height),+ };+ const href = appendProxyParam(copy.originalSrc, copy.variant, box);+ if (pendingUpgrades.get(el) === href) return;+ pendingUpgrades.set(el, href);+ const wanted = () =>+ pendingUpgrades.get(el) === href && el.isConnected && el.getAttribute("src") === copy.href;+ void waitForServedProxy(href, wanted).then((served) => {+ if (served && wanted()) loadProxy(el, href, copy.originalSrc, copy.variant, box);+ if (pendingUpgrades.get(el) === href) pendingUpgrades.delete(el);+ });+}++function loadProxy(+ el: HTMLMediaElement,+ href: string,+ originalSrc: string,+ variant: string,+ box: PreviewProxyBox | null,+): void {+ if (box) boxedElements.set(el, { originalSrc, variant, href, box });+ else boxedElements.delete(el);+ // Sync state measured on the previous file would read the new file's buffering as drift.+ evictMediaSyncState(el);+ el.src = href;+ el.load();+}+/*** Render mode never proxies: the codec map arrives by injection (not a* fetch, so determinism holds) but render always decodes the original via@@ -164,9 +253,10 @@ function lookupCodecMapEntry();}-function appendProxyParam(src: string, entry: MediaCodecMapEntry | null): string {+function appendProxyParam(src: string, variant: string, box: PreviewProxyBox | null): string {const url = new URL(src, document.baseURI);- url.searchParams.set(PROXY_QUERY_PARAM, entry ? (entry.hasAlpha ? "vp8" : "h264") : "auto");+ url.searchParams.set(PROXY_QUERY_PARAM, variant);+ if (box) url.searchParams.set(PREVIEW_PROXY_BOX_PARAM, formatPreviewProxyBox(box));return url.href;}@@ -223,9 +313,9 @@ export function swapToProxy(if (swappedElements.has(el)) return;if (proxyRequested.has(el) && proxyRequested.get(el) === el.getAttribute("src")) return;const originalSrc = currentSrcValue(el);- let proxiedSrc: string;+ const variant = entry ? (entry.hasAlpha ? "vp8" : "h264") : "auto";try {- proxiedSrc = appendProxyParam(originalSrc, entry);+ new URL(originalSrc, document.baseURI);} catch (err) {swallow("runtime.mediaProxy.swap", err);emitUnavailableDiagnostic(el, "invalid_source_url", originalSrc);@@ -234,7 +324,11 @@ export function swapToProxy(const originalAttr = el.getAttribute("src");proxyRequested.set(el, originalAttr);const live = () => el.isConnected && el.getAttribute("src") === originalAttr;- const swap = waitForServedProxy(proxiedSrc, live).then((served) => {+ const swap = shownScale().then(async (scale) => {+ // Asked once the shown size is known, so the copy is made at that size and only once.+ const box = scale === null ? null : shownBox(el, scale);+ const proxiedSrc = appendProxyParam(originalSrc, variant, box);+ const served = await waitForServedProxy(proxiedSrc, live);if (!live()) {if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el);return;@@ -244,10 +338,9 @@ export function swapToProxy(return;}swappedElements.set(el, originalAttr);- // Sync state measured on the original would read the new file's buffering as drift.- evictMediaSyncState(el);- el.src = proxiedSrc;- el.load();+ loadProxy(el, proxiedSrc, originalSrc, variant, box);+ // The shown size may have grown while this copy was being made.+ if (displayScale !== null) keepSized(el, displayScale);return new Promise((landed) => {el.addEventListener("loadeddata", landed, { once: true });el.addEventListener("error", landed, { once: true });diff --git a/packages/core/src/runtime/protocol.ts b/packages/core/src/runtime/protocol.tsindex 894f9590bb..71ed37cd43 100644--- a/packages/core/src/runtime/protocol.ts+++ b/packages/core/src/runtime/protocol.ts@@ -70,6 +70,12 @@ export function runtimeProtocolMetadata(fps: number): RuntimeProtocolV1 {};}+export function frameDisplayScale(frame: HTMLElement): number | null {+ const layoutWidth = frame.offsetWidth;+ const shownWidth = frame.getBoundingClientRect().width;+ return layoutWidth > 0 && shownWidth > 0 ? shownWidth / layoutWidth : null;+}+function hasDeclaredCapabilities(value: unknown): boolean {return Array.isArray(value) && value.every((capability) => typeof capability === "string");}diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.tsindex 25a16a2b19..fe70857f01 100644--- a/packages/core/src/runtime/types.ts+++ b/packages/core/src/runtime/types.ts@@ -21,6 +21,7 @@ type RuntimeBridgeControlActionBase =| "set-native-media-sync-disabled"| "set-web-audio-media-disabled"| "set-idle-heartbeat"+ | "set-display-scale"| "set-root-duration"| "set-play-range"| "stop-media"@@ -39,6 +40,7 @@ type RuntimeBridgeControlMessageBase = {endSeconds?: number | null;disabled?: boolean;slow?: boolean;+ scale?: number;playbackRate?: number;target?: HfColorGradingTarget | string | null;grading?: RuntimeJson;diff --git a/packages/player/src/hyperframes-player.test.ts b/packages/player/src/hyperframes-player.test.tsindex 481a1cb689..95430f9079 100644--- a/packages/player/src/hyperframes-player.test.ts+++ b/packages/player/src/hyperframes-player.test.ts@@ -2348,6 +2348,25 @@ describe("HyperframesPlayer runtime ready handshake", () => {expect(findControlCalls("set-idle-heartbeat")[0]?.[0]).toMatchObject({ slow: false });});+ it("tells the runtime how large the frame shows, on fit and again on play", () => {+ Object.defineProperty(player, "offsetWidth", { configurable: true, value: 540 });+ Object.defineProperty(player, "offsetHeight", { configurable: true, value: 960 });+ // The frame's on-screen box, after the fit and any zoom the page puts around the player,+ // over its layout width, which a default size can hold before the stage size arrives.+ let shownWidth = 540;+ player.iframe.getBoundingClientRect = () => ({ width: shownWidth }) as DOMRect;+ Object.defineProperty(player.iframe, "offsetWidth", { configurable: true, value: 1080 });+ postSpy.mockClear();++ player._onMessage(stageSizeMessage(1080, 1920));+ expect(findControlCalls("set-display-scale").at(-1)?.[0]).toMatchObject({ scale: 0.5 });++ shownWidth = 1080;+ postSpy.mockClear();+ player.play();+ expect(findControlCalls("set-display-scale")[0]?.[0]).toMatchObject({ scale: 1 });+ });+it("keeps runtime WebAudio media enabled outside slideshow embeds", () => {postSpy.mockClear();diff --git a/packages/player/src/hyperframes-player.ts b/packages/player/src/hyperframes-player.tsindex 8567b727ea..d09b78086f 100644--- a/packages/player/src/hyperframes-player.ts+++ b/packages/player/src/hyperframes-player.ts@@ -26,7 +26,7 @@ import { ShaderLoaderState } from "./shader-loader-state.js";import { PLAYER_STYLES } from "./styles.js";import { type DirectTimelineAdapter } from "./timeline-adapters.js";import { createVideoSource, isVideoType, type VideoSource } from "./video-source.js";-import { runtimeProtocolMetadata } from "@hyperframes/core/runtime/protocol";+import { frameDisplayScale, runtimeProtocolMetadata } from "@hyperframes/core/runtime/protocol";import {FIRST_FRAME_READINESS_SCOPE,scanPendingCompositionAssets,@@ -431,6 +431,7 @@ class HyperframesPlayer extends HTMLElement {// starts playback — this premature call must not ALSO dispatch "play"// for what hasn't started, or a host listener sees it fire twice.let queuedForReady = false;+ this._sendDisplayScale();if (!directTimelineStarted) {this._sendControl("play");// Only start the parent tick clock once the composition is ready and@@ -1002,6 +1003,12 @@ class HyperframesPlayer extends HTMLElement {disabled: this._isSlideshowPlayer(),});this._sendControl("set-idle-heartbeat", { slow: this.hasAttribute(LOW_POWER_IDLE_ATTR) });+ this._sendDisplayScale();+ }++ private _sendDisplayScale(): void {+ const scale = frameDisplayScale(this.iframe);+ if (scale) this._sendControl("set-display-scale", { scale });}private _reloadShaderOptions(): void {@@ -1538,6 +1545,7 @@ class HyperframesPlayer extends HTMLElement {// why in the field. Surface it once (not on every ResizeObserver tick —// a legitimately hidden/zero-sized player, e.g. a collapsed tab or// off-screen carousel card, would otherwise spam the console forever).+ if (applied) this._sendDisplayScale();if (!applied && this._ready && !this._rescaleWarned) {this._rescaleWarned = true;console.warn("[hyperframes-player] rescale no-op after ready — zero-size player element", {diff --git a/packages/studio-server/src/helpers/mediaProxyPreview.ts b/packages/studio-server/src/helpers/mediaProxyPreview.tsindex 74efe614f6..f76e683b6a 100644--- a/packages/studio-server/src/helpers/mediaProxyPreview.ts+++ b/packages/studio-server/src/helpers/mediaProxyPreview.ts@@ -1,4 +1,5 @@import { resolve } from "node:path";+import { formatPreviewProxyBox, type PreviewProxyBox } from "@hyperframes/core";import { insertBeforeCloseTag } from "@hyperframes/core/compiler/html-document";import type { StudioApiAdapter } from "../types.js";import {@@ -62,9 +63,9 @@ export function resolvePreviewMediaCodecProbeCache(* or a different proxy variant invalidates cached 304s without needing to* touch the proxy file itself.*/-export function proxyEtagSalt(raw: string | undefined): string {+export function proxyEtagSalt(raw: string | undefined, box?: PreviewProxyBox): string {if (raw === undefined) return "";- return `:proxy:${raw}:${PROXY_PARAMS_VERSION}`;+ return `:proxy:${raw}:${PROXY_PARAMS_VERSION}${box ? `:${formatPreviewProxyBox(box)}` : ""}`;}/**@@ -91,6 +92,7 @@ export async function injectMediaCodecMapIntoHtml(projectDir: string,htmlSources: HtmlSourceLike[],probeCache?: MediaCodecProbeCache,+ prewarm = true,): Promise<string> {let map: MediaCodecMap;try {@@ -105,7 +107,7 @@ export async function injectMediaCodecMapIntoHtml(}if (Object.keys(map).length === 0) return html;for (const [rootRelativePathname, facts] of Object.entries(map)) {- if (!shouldPrewarmProxy(facts)) continue;+ if (!prewarm || !shouldPrewarmProxy(facts)) continue;recordProxyPrewarm();resolveProxy(projectDir,@@ -138,5 +140,5 @@ export async function injectMediaCodecMap(probeCache: MediaCodecProbeCache,): Promise<string> {if (!isAutoProxyEnabled(adapter)) return html;- return injectMediaCodecMapIntoHtml(html, projectDir, [{ html, compSrcPath }], probeCache);+ return injectMediaCodecMapIntoHtml(html, projectDir, [{ html, compSrcPath }], probeCache, false);}diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.tsindex a209cfefbd..4519598b44 100644--- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts+++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts@@ -417,6 +417,43 @@ describe("resolveProxy", () => {60_000,);+ it.skipIf(!realFfmpeg || !findFfBinary("ffprobe"))(+ "makes a preview copy that fills the shown box, never larger than the source",+ async () => {+ vi.resetModules();+ vi.doUnmock("./mediaMetadata.js");+ const { resolveProxy } = await import("./proxyTranscoder.js");+ const projectDir = tmpProject();+ const sourcePath = join(projectDir, "portrait.mp4");+ execFileSync(realFfmpeg!, [+ ...["-v", "error", "-f", "lavfi", "-i", "testsrc2=s=1080x1920:d=0.1"],+ ...["-c:v", "libx264", "-pix_fmt", "yuv420p", sourcePath],+ ]);+ const size = (path: string) =>+ execFileSync(+ findFfBinary("ffprobe")!,+ [+ ...["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height"],+ ...["-of", "csv=p=0", "--", path],+ ],+ { encoding: "utf8" },+ ).trim();++ // The width side binds: 512/1080 > 724/1920, so the height follows the source's shape.+ const boxed = await resolveProxy(projectDir, sourcePath, "h264", { width: 512, height: 724 });+ expect(size(boxed)).toBe("512,912");+ const roomy = await resolveProxy(projectDir, sourcePath, "h264", {+ width: 2048,+ height: 2048,+ });+ expect(size(roomy)).toBe("1080,1920");+ const full = await resolveProxy(projectDir, sourcePath);+ expect(size(full)).toBe("1080,1920");+ expect(new Set([boxed, roomy, full]).size).toBe(3);+ },+ 60_000,+ );+it("dedupes two concurrent same-key calls to one spawn", async () => {const { spawn, calls } = createSpawnSpy();const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH);diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.tsindex 85d205e980..e585b11fae 100644--- a/packages/studio-server/src/helpers/proxyTranscoder.ts+++ b/packages/studio-server/src/helpers/proxyTranscoder.ts@@ -2,7 +2,11 @@ import { spawn } from "node:child_process";import { createHash, randomUUID } from "node:crypto";import { existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs";import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";-import { hdrToSdrToneMapFilter } from "@hyperframes/core";+import {+ formatPreviewProxyBox,+ hdrToSdrToneMapFilter,+ type PreviewProxyBox,+} from "@hyperframes/core";import { findFfBinary } from "@hyperframes/parsers/ff-binaries";import { probeFirstFrameColour, probeMediaMetadata } from "./mediaMetadata.js";import { cleanupProxyCache } from "./proxyCache.js";@@ -174,17 +178,26 @@ function canonicalizeProxySource(};}-function buildProxyCacheKey(source: CanonicalProxySource, variant: ProxyVariant): string {+function buildProxyCacheKey(+ source: CanonicalProxySource,+ variant: ProxyVariant,+ box: PreviewProxyBox | undefined,+): string {const stat = statSync(source.sourcePath);+ const size = box ? `\0${formatPreviewProxyBox(box)}` : "";return createHash("sha256").update(- `${source.relativePath}\0${source.cacheIdentity}\0${stat.mtimeMs}\0${stat.size}\0${PROXY_PARAMS_VERSION}\0${variant}`,+ `${source.relativePath}\0${source.cacheIdentity}\0${stat.mtimeMs}\0${stat.size}\0${PROXY_PARAMS_VERSION}\0${variant}${size}`,).digest("hex");}-function getCanonicalProxyCachePath(source: CanonicalProxySource, variant: ProxyVariant): string {- const key = buildProxyCacheKey(source, variant);+function getCanonicalProxyCachePath(+ source: CanonicalProxySource,+ variant: ProxyVariant,+ box?: PreviewProxyBox,+): string {+ const key = buildProxyCacheKey(source, variant, box);return join(source.projectDir,CACHE_DIR_NAME,@@ -324,10 +337,19 @@ export function clearFailedTranscodesForTest(): void {failedTranscodes.clear();}+/** Even dimensions; with a box, the source shrinks until its tighter side fills the box, never grows. */+function proxyScaleFilter(box?: PreviewProxyBox): string {+ if (!box) return "scale=trunc(iw/2)*2:trunc(ih/2)*2";+ const factor = `min(1\\,max(${box.width}/iw\\,${box.height}/ih))`;+ const side = (d: string) => `min(trunc(${d}/2)*2\\,ceil(${d}*${factor}/2)*2)`;+ return `scale=${side("iw")}:${side("ih")}`;+}+async function runFfmpeg(sourcePath: string,outputPath: string,variant: ProxyVariant,+ box: PreviewProxyBox | undefined,): Promise<void> {const metadata = await probeMediaMetadata(sourcePath);const ffmpegPath = findFfBinary("ffmpeg", { configuredMustExist: true });@@ -339,7 +361,7 @@ async function runFfmpeg(const toneMap = (hdrTransfer === "pq" || hdrTransfer === "hlg") && !keepsAlpha;if (toneMap) await ensureHdrFilters(ffmpegPath);const firstFrame = toneMap ? await probeFirstFrameColour(sourcePath) : {};- const evenScale = "scale=trunc(iw/2)*2:trunc(ih/2)*2";+ const evenScale = proxyScaleFilter(box);const pixelFormat = keepsAlpha ? "yuva420p" : "yuv420p";// The tone map ends in RGB; older ffmpeg (seen on 5.1) converts it with BT.601 unless the matrix is named.const videoFilter = toneMap@@ -444,6 +466,7 @@ async function transcodeToCache(absoluteSourcePath: string,cachePath: string,variant: ProxyVariant,+ box: PreviewProxyBox | undefined,): Promise<string> {await acquireSlot();try {@@ -454,7 +477,7 @@ async function transcodeToCache(mkdirWithinProject(projectDir, cacheDir);const tempPath = join(cacheDir, `.tmp-${randomUUID()}-${basename(cachePath)}`);try {- await runFfmpeg(absoluteSourcePath, tempPath, variant);+ await runFfmpeg(absoluteSourcePath, tempPath, variant, box);renameSync(tempPath, cachePath);maintainProxyCache(cacheDir);return cachePath;@@ -485,14 +508,17 @@ export function proxyActivityMark(projectDir: string): string | null {* one promise; calls for different keys queue through the global concurrency* limiter above. Throws `ProxyTranscodeError` on failure (missing ffmpeg or a* nonzero exit) — callers (route handlers) decide how to surface that (502).+ * A `box` makes a smaller preview-only copy with its own cache entry; without+ * one the copy keeps the source size (CLI play, static servers, publish).*/export async function resolveProxy(projectDir: string,absoluteSourcePath: string,variant: ProxyVariant = "h264",+ box?: PreviewProxyBox,): Promise<string> {const source = canonicalizeProxySource(projectDir, absoluteSourcePath);- const cachePath = getCanonicalProxyCachePath(source, variant);+ const cachePath = getCanonicalProxyCachePath(source, variant, box);if (existsSync(cachePath)) {markCacheEntryUsed(cachePath);maintainProxyCache(dirname(cachePath));@@ -508,7 +534,7 @@ export async function resolveProxy(const existing = inFlight.get(cachePath);if (existing) return existing;- const promise = transcodeToCache(source.projectDir, source.sourcePath, cachePath, variant)+ const promise = transcodeToCache(source.projectDir, source.sourcePath, cachePath, variant, box).catch((err: unknown) => {if (!(err instanceof ProxyCapacityError)) rememberFailure(cachePath, err);throw err;diff --git a/packages/studio-server/src/routes/preview.test.ts b/packages/studio-server/src/routes/preview.test.tsindex 5a804081cb..cfab3342c6 100644--- a/packages/studio-server/src/routes/preview.test.ts+++ b/packages/studio-server/src/routes/preview.test.ts@@ -1316,6 +1316,7 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {projectDir: string,absoluteSourcePath: string,variant?: "h264" | "vp8",+ box?: { width: number; height: number },) => Promise<string>;scanMapImpl?: ScanMapImpl;probeAssetCodecImpl?: () => Promise<{@@ -1426,6 +1427,7 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {projectDir,join(projectDir, "clip.mp4"),"h264",+ undefined,);const ranged = await app.request(@@ -1472,6 +1474,62 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {expect(resolveProxyMock).toHaveBeenCalledTimes(1);});+ it("makes the copy for the shown size, under its own tag", async () => {+ const projectDir = createProjectDir();+ writeFileSync(join(projectDir, "clip.mp4"), "original-hevc-bytes");+ pastSettleWindow();+ const resolveProxyMock = vi.fn(async () => {+ const proxyPath = join(projectDir, "proxy.mp4");+ writeFileSync(proxyPath, "proxy-bytes");+ return proxyPath;+ });+ const { registerPreviewRoutes: register } = await loadPreviewModule({+ resolveProxyImpl: resolveProxyMock,+ });+ const app = new Hono();+ register(app, createAdapter(projectDir));++ const sized = await app.request(+ "http://localhost/projects/demo/preview/clip.mp4?hf-proxy=h264&hf-proxy-box=1448x2048",+ );+ expect(sized.status).toBe(200);+ expect(resolveProxyMock).toHaveBeenLastCalledWith(+ projectDir,+ join(projectDir, "clip.mp4"),+ "h264",+ { width: 1448, height: 2048 },+ );+ const full = await app.request(+ "http://localhost/projects/demo/preview/clip.mp4?hf-proxy=h264",+ );+ expect(resolveProxyMock).toHaveBeenLastCalledWith(+ projectDir,+ join(projectDir, "clip.mp4"),+ "h264",+ undefined,+ );+ expect(sized.headers.get("ETag")).toBeTruthy();+ expect(sized.headers.get("ETag")).not.toBe(full.headers.get("ETag"));+ });++ it.each([+ ["a size off the ladder", "?hf-proxy=h264&hf-proxy-box=1000x2000"],+ ["a size without a copy", "?hf-proxy-box=1448x2048"],+ ])("refuses %s before making anything", async (_name, query) => {+ const projectDir = createProjectDir();+ writeFileSync(join(projectDir, "clip.mp4"), "original-hevc-bytes");+ const resolveProxyMock = vi.fn(async () => "should-not-be-called");+ const { registerPreviewRoutes: register } = await loadPreviewModule({+ resolveProxyImpl: resolveProxyMock,+ });+ const app = new Hono();+ register(app, createAdapter(projectDir));++ const res = await app.request(`http://localhost/projects/demo/preview/clip.mp4${query}`);+ expect(res.status).toBe(404);+ expect(resolveProxyMock).not.toHaveBeenCalled();+ });+it("tags no asset written in the last moments, so a same-size rewrite cannot reuse its tag", async () => {const projectDir = createProjectDir();writeFileSync(join(projectDir, "clip.mp4"), "original-hevc-bytes");@@ -1579,6 +1637,7 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {projectDir,join(projectDir, "clip.mov"),"vp8",+ undefined,);});@@ -1767,7 +1826,7 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {const html = await res.text();expect(html).not.toContain("data-hf-media-codec-map");});- it("injects the scanned map naming the hostile fixture, and pre-warms resolveProxy for it", async () => {+ it("injects the scanned map naming the hostile fixture, and makes no copy before the page asks for its size", async () => {const projectDir = createProjectDir();const resolveProxyMock = vi.fn(async () => join(projectDir, ".transcode-cache", "x.mp4"));const scanMapMock = vi.fn(async () => ({@@ -1793,14 +1852,8 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {expect(html).not.toContain("h264.mp4");expect(scanMapMock).toHaveBeenCalled();- // Pre-warm: fire-and-forget resolveProxy for the hostile entry.- await Promise.resolve();- await Promise.resolve();- expect(resolveProxyMock).toHaveBeenCalledWith(- projectDir,- join(projectDir, "/videos/hevc.mp4"),- "h264",- );+ // A source-size pre-warm would be a copy Studio never shows.+ expect(resolveProxyMock).not.toHaveBeenCalled();});it("injects and serves a proxy for a hostile video through an external asset symlink", async () => {@@ -1845,6 +1898,7 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {projectDir,join(projectDir, "assets", "shared", "clip.mov"),"h264",+ undefined,);});diff --git a/packages/studio-server/src/routes/preview.ts b/packages/studio-server/src/routes/preview.tsindex 7c2865f553..b1a026b2a6 100644--- a/packages/studio-server/src/routes/preview.ts+++ b/packages/studio-server/src/routes/preview.ts@@ -17,6 +17,7 @@ import {STUDIO_PREVIEW_MARK_META,STUDIO_PREVIEW_ERRORS,} from "@hyperframes/core/studio-preview-mark";+import { PREVIEW_PROXY_BOX_PARAM, parsePreviewProxyBox } from "@hyperframes/core";import { gsapCdnDist, motionPathPluginUrl } from "@hyperframes/core/gsap-cdn";import { findStartTags, injectTagsAtHeadStart } from "@hyperframes/core/compiler/html-document";import { isWithinProjectRoot } from "@hyperframes/parsers/asset-resolution";@@ -648,6 +649,11 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo// for this adapter/project. Checked BEFORE any transcode or 304 shortcut// so a bogus/disabled request never spawns ffmpeg.const proxyParam = c.req.query("hf-proxy");+ const boxParam = c.req.query(PREVIEW_PROXY_BOX_PARAM);+ const proxyBox = boxParam === undefined ? undefined : parsePreviewProxyBox(boxParam);+ if (proxyBox === null || (proxyBox && proxyParam === undefined)) {+ return c.text("not found", 404);+ }let proxyVariant: ProxyVariant | undefined;if (proxyParam !== undefined) {if (@@ -670,7 +676,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo}const tag = settledFileTag(stat);- const etag = tag && `"${tag}${proxyEtagSalt(proxyVariant)}"`;+ const etag = tag && `"${tag}${proxyEtagSalt(proxyVariant, proxyBox)}"`;const cacheHeaders: Record<string, string> = isText? { "Cache-Control": "no-store" }: { "Cache-Control": "private, no-cache", ...(etag && { ETag: etag }) };@@ -691,7 +697,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): votry {// A cached copy settles before any timer; a transcode never holds one of// the browser's few connections to this host. 202 until the copy lands.- servedPath = await waitForProxy(resolveProxy(project.dir, file, proxyVariant), 0);+ servedPath = await waitForProxy(resolveProxy(project.dir, file, proxyVariant, proxyBox), 0);} catch (err) {if (err instanceof ProxyWaitTimeoutError) {return c.text("media proxy is being made", 202, {diff --git a/packages/studio/src/components/nle/NLEPreview.test.ts b/packages/studio/src/components/nle/NLEPreview.test.tsindex 6f61f8e671..cc43611f76 100644--- a/packages/studio/src/components/nle/NLEPreview.test.ts+++ b/packages/studio/src/components/nle/NLEPreview.test.ts@@ -344,6 +344,22 @@ describe("NLEPreview", () => {view.cleanup();});+ it("tells the preview how large it shows once a zoom settles, which the player cannot see", () => {+ const view = renderPreview();+ const frame = view.host.querySelector<HTMLElement>('[data-testid="mock-player"]')!;+ const postMessage = vi.fn();+ Object.defineProperty(frame, "contentWindow", { value: { postMessage } });+ Object.defineProperty(frame, "offsetWidth", { value: 800 });+ frame.getBoundingClientRect = () => ({ width: 1600 }) as DOMRect;++ pinchIn(view, 10);+ expect(postMessage).toHaveBeenLastCalledWith(+ expect.objectContaining({ action: "set-display-scale", scale: 2 }),+ "*",+ );+ view.cleanup();+ });+it("says how far it is zoomed, shows where in the frame, and Fit puts it back", () => {const view = renderPreview();expect([chip(view), navigator(view)]).toEqual([null, null]);diff --git a/packages/studio/src/components/nle/NLEPreview.tsx b/packages/studio/src/components/nle/NLEPreview.tsxindex e41c77e7c2..8cb82c9dc0 100644--- a/packages/studio/src/components/nle/NLEPreview.tsx+++ b/packages/studio/src/components/nle/NLEPreview.tsx@@ -12,6 +12,7 @@ import {type RefObject,} from "react";import { Player } from "../../player";+import { postFrameDisplayScale } from "../../player/lib/runtimeProtocol";import type { NLEContextValue } from "./NLEContext";import {DEFAULT_PREVIEW_ZOOM,@@ -158,6 +159,7 @@ export const NLEPreview = memo(function NLEPreview({const zoomRef = useRef<PreviewZoomState>(DEFAULT_PREVIEW_ZOOM);const [settledZoom, setSettledZoom] = useState<PreviewZoomState>(DEFAULT_PREVIEW_ZOOM);+ useEffect(() => postFrameDisplayScale(previewIframeRef.current), [settledZoom]);const hudRef = useRef<HTMLDivElement>(null);const hudTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);const settleTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);diff --git a/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsx b/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsxindex efe71972c8..8786920867 100644--- a/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsx+++ b/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsx@@ -247,6 +247,49 @@ describe("composition card thumbnails", () => {expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:frame-2");});+ it("tell a hover preview how large it shows, so its videos are not copied at source size", () => {+ vi.useFakeTimers();+ try {+ const host = mount();+ const card = host.querySelector<HTMLElement>('[aria-label^="Open composition"]')!;+ act(() => {+ card.dispatchEvent(+ new MouseEvent("pointerover", { bubbles: true, relatedTarget: document.body }),+ );+ });+ act(() => vi.advanceTimersByTime(300));+ const frame = host.querySelector("iframe")!;+ const postMessage = vi.fn();+ Object.defineProperty(frame, "contentWindow", { value: { postMessage } });+ Object.defineProperty(frame, "offsetWidth", { value: 1920 });+ frame.getBoundingClientRect = () => ({ width: 80 }) as DOMRect;+ act(() => {+ frame.dispatchEvent(new Event("load"));+ });+ expect(postMessage).toHaveBeenCalledWith(+ expect.objectContaining({ action: "set-display-scale", scale: 80 / 1920 }),+ "*",+ );++ // A module-script composition starts listening only after load, then says it is ready.+ postMessage.mockClear();+ act(() => {+ window.dispatchEvent(+ new MessageEvent("message", {+ source: frame.contentWindow,+ data: { source: "hf-preview", type: "ready" },+ }),+ );+ });+ expect(postMessage).toHaveBeenCalledWith(+ expect.objectContaining({ action: "set-display-scale", scale: 80 / 1920 }),+ "*",+ );+ } finally {+ vi.useRealTimers();+ }+ });+it("show a fallback when the render fails, and retry it at the next content revision", async () => {const host = mount();renders[0]!.answer(500);diff --git a/packages/studio/src/components/sidebar/CompositionsTab.tsx b/packages/studio/src/components/sidebar/CompositionsTab.tsxindex 9eb923b991..171edc91e9 100644--- a/packages/studio/src/components/sidebar/CompositionsTab.tsx+++ b/packages/studio/src/components/sidebar/CompositionsTab.tsx@@ -13,6 +13,7 @@ import {THUMBNAIL_SEEK_TIME_SECONDS,} from "../../player/components/CompositionThumbnail";import { setPreviewMediaMuted } from "../../player/lib/timelineIframeHelpers";+import { postFrameDisplayScale } from "../../player/lib/runtimeProtocol";import { usePlayerStore } from "../../player/store/playerStore";import { thumbnailRevisionOf } from "../../player/store/thumbnailSlice";import { encodePreviewPath } from "../../player/components/thumbnailUtils";@@ -258,6 +259,22 @@ function CompCard({if (hovered) requestIframePlaybackSync(true);}, [hovered, requestIframePlaybackSync]);+ useEffect(() => {+ if (livePreviewLoaded) postFrameDisplayScale(iframeRef.current);+ }, [livePreviewLoaded, previewScale]);++ useEffect(() => {+ if (!hovered) return;+ const onRuntimeReady = (event: MessageEvent) => {+ const frame = iframeRef.current;+ if (frame && event.source === frame.contentWindow && event.data?.type === "ready") {+ postFrameDisplayScale(frame);+ }+ };+ window.addEventListener("message", onRuntimeReady);+ return () => window.removeEventListener("message", onRuntimeReady);+ }, [hovered]);+useEffect(() => {return () => {if (hoverTimer.current) clearTimeout(hoverTimer.current);diff --git a/packages/studio/src/player/lib/runtimeProtocol.ts b/packages/studio/src/player/lib/runtimeProtocol.tsindex 7aeccb582a..1084661159 100644--- a/packages/studio/src/player/lib/runtimeProtocol.ts+++ b/packages/studio/src/player/lib/runtimeProtocol.ts@@ -1,4 +1,5 @@import {+ frameDisplayScale,inspectRuntimeProtocol,runtimeProtocolMetadata,type RuntimeProtocolInspection,@@ -34,6 +35,11 @@ export function postRuntimeControlMessage(target?.postMessage(createRuntimeControlMessage(action, payload, fps), "*");}+export function postFrameDisplayScale(iframe: HTMLIFrameElement | null | undefined): void {+ const scale = iframe && frameDisplayScale(iframe);+ if (scale) postRuntimeControlMessage(iframe.contentWindow, "set-display-scale", { scale });+}+export function inspectStudioRuntimeMessage(value: unknown): RuntimeProtocolInspection {return inspectRuntimeProtocol(value, 30);}