Skip to content

Write HTML. Render video. Built for agents.

fix: the preview preloads the coming scene's images, and check stops failing on image swaps (#5128)

Miguel Ángel11e684d3050f
diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts
index d28011c78a..c67a35cfa0 100644
--- a/packages/cli/src/utils/checkBrowser.test.ts
+++ b/packages/cli/src/utils/checkBrowser.test.ts
@@ -11,6 +11,7 @@ import {
import { DEFAULT_CHECK_OPTIONS, runAuditGrid } from "./checkPipeline.js";
import {
captureOverviewShot,
+ keepBrokenImageAborts,
preResolveHostileMediaProxies,
runBrowserCheck,
} from "./checkBrowser.js";
@@ -83,6 +84,7 @@ const PROJECT: ProjectDir = {
};
afterEach(() => {
+ vi.useRealTimers();
vi.clearAllMocks();
vi.restoreAllMocks();
vi.unstubAllGlobals();
@@ -526,6 +528,95 @@ it("elevates and deduplicates WebGPU validation warnings while preserving ordina
);
});
+describe("keepBrokenImageAborts", () => {
+ const draft = (url: string, abortedImage: boolean) => ({
+ code: "request_failed",
+ severity: "error" as const,
+ message: `Failed to load ${url}`,
+ time: 0,
+ url,
+ abortedImage,
+ });
+
+ it("keeps an aborted image only when it is still broken, and every other failure", () => {
+ const swappedPast = draft("http://h/seq/0004.png", true);
+ const stuck = draft("http://h/plate.png", true);
+ const missing = draft("http://h/gone.png", false);
+ const kept = keepBrokenImageAborts(
+ [swappedPast, stuck, missing],
+ new Set(["http://h/plate.png"]),
+ );
+ expect(kept).toEqual([stuck, missing]);
+ });
+
+ it("reports an aborted image a page still shows broken, not one swapped away or one that decodes", async () => {
+ const base = "http://127.0.0.1:3000/assets";
+ mountCanvasFixture(
+ `<img id="stuck" src="${base}/plate.png"><img id="fine" src="${base}/seq/0012.png">`,
+ );
+ const stuck = document.getElementById("stuck") as HTMLImageElement;
+ const fine = document.getElementById("fine") as HTMLImageElement;
+ stuck.decode = () => Promise.reject(new Error("broken"));
+ fine.decode = () => Promise.resolve();
+ const aborted = (url: string) => ({
+ url: () => url,
+ failure: () => ({ errorText: "net::ERR_ABORTED" }),
+ resourceType: () => "image",
+ });
+ const page = fakePage();
+ page.on = vi.fn((event: string, handler: (request: ReturnType<typeof aborted>) => void) => {
+ if (event !== "requestfailed") return;
+ for (const url of ["seq/0004.png", "seq/0012.png", "plate.png"])
+ handler(aborted(`${base}/${url}`));
+ });
+ installSessionMock(page);
+
+ const result = await runBrowserCheck(
+ PROJECT,
+ { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false },
+ { kind: "none" },
+ runAuditGrid,
+ );
+
+ const failed = result.runtimeFindings.filter((finding) => finding.code === "request_failed");
+ expect(failed.map((finding) => finding.message)).toEqual([
+ "Failed to load assets/plate.png: net::ERR_ABORTED",
+ ]);
+ });
+
+ it("does not report an aborted image whose reload is still pending at the cap", async () => {
+ const url = "http://127.0.0.1:3000/assets/slow.png";
+ mountCanvasFixture(`<img id="slow" src="${url}">`);
+ (document.getElementById("slow") as HTMLImageElement).decode = () => new Promise(() => {});
+ const page = fakePage();
+ page.on = vi.fn((event: string, handler: (request: unknown) => void) => {
+ if (event !== "requestfailed") return;
+ handler({
+ url: () => url,
+ failure: () => ({ errorText: "net::ERR_ABORTED" }),
+ resourceType: () => "image",
+ });
+ });
+ installSessionMock(page);
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ try {
+ const checked = runBrowserCheck(
+ PROJECT,
+ { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false },
+ { kind: "none" },
+ runAuditGrid,
+ );
+ await vi.runAllTimersAsync();
+ const result = await checked;
+ expect(result.runtimeFindings.filter((finding) => finding.code === "request_failed")).toEqual(
+ [],
+ );
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
+
describe("preResolveHostileMediaProxies", () => {
const dirs: string[] = [];
const mkProjectDir = (): string => {
diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts
index d7bb3cf438..092448829e 100644
--- a/packages/cli/src/utils/checkBrowser.ts
+++ b/packages/cli/src/utils/checkBrowser.ts
@@ -63,6 +63,7 @@ interface RuntimeDraft {
url?: string;
line?: number;
count?: number;
+ abortedImage?: boolean;
}
interface AnchorRequest {
@@ -195,10 +196,13 @@ export async function runBrowserCheck(
currentTime = time;
});
const result = await runGrid(driver, options, motion);
+ const broken = await abortedImagesStillBroken(page, drafts);
return {
...result,
timings: { ...result.timings, launchSettleMs },
- runtimeFindings: drafts.map((draft) => runtimeFinding(draft, rootAnchor)),
+ runtimeFindings: keepBrokenImageAborts(drafts, broken).map((draft) =>
+ runtimeFinding(draft, rootAnchor),
+ ),
};
} finally {
await chromeBrowser?.close().catch(() => undefined);
@@ -367,6 +371,46 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
wireNetworkListeners(page, drafts, currentTime);
}
+/** Check's scrubs cancel image loads: an aborted image failed only if an `<img>` shows it and its decode fails. */
+export function keepBrokenImageAborts(drafts: RuntimeDraft[], broken: Set<string>): RuntimeDraft[] {
+ return drafts.filter((draft) => !draft.abortedImage || broken.has(draft.url ?? ""));
+}
+
+const IMAGE_DECODE_CAP_MS = 5000;
+
+async function abortedImagesStillBroken(page: Page, drafts: RuntimeDraft[]): Promise<Set<string>> {
+ const urls = drafts.filter((draft) => draft.abortedImage).map((draft) => draft.url ?? "");
+ if (urls.length === 0) return new Set();
+ const broken = await page.evaluate(
+ async (candidates: string[], capMs: number) => {
+ // A load still pending at the cap (a deferred lazy image) is not a failure.
+ const fails = (img: HTMLImageElement) =>
+ new Promise<boolean>((resolve) => {
+ const cap = setTimeout(() => resolve(false), capMs);
+ img
+ .decode()
+ .then(
+ () => resolve(false),
+ () => resolve(true),
+ )
+ .finally(() => clearTimeout(cap));
+ });
+ const stillBroken = await Promise.all(
+ candidates.map(async (url) => {
+ const shown = Array.from(document.querySelectorAll("img")).filter(
+ (img) => img.currentSrc === url || img.src === url,
+ );
+ return (await Promise.all(shown.map(fails))).some(Boolean);
+ }),
+ );
+ return candidates.filter((_, i) => stillBroken[i]);
+ },
+ urls,
+ IMAGE_DECODE_CAP_MS,
+ );
+ return new Set(broken);
+}
+
function wireNetworkListeners(page: Page, drafts: RuntimeDraft[], currentTime: () => number): void {
page.on("requestfailed", (request) => {
const url = request.url();
@@ -379,6 +423,7 @@ function wireNetworkListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
message: `Failed to load ${urlPath(url)}: ${failure ?? "net::ERR_FAILED"}`,
time: currentTime(),
url,
+ abortedImage: failure === "net::ERR_ABORTED" && request.resourceType() === "image",
});
});
page.on("response", (response) => {
diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts
index 9398089799..dbce99d103 100644
--- a/packages/core/src/runtime/entry.test.ts
+++ b/packages/core/src/runtime/entry.test.ts
@@ -197,6 +197,34 @@ describe("runtime entry", () => {
expect(imageSkipped(later)).toEqual([false]);
});
+ it("loads the next scene's images within the look-ahead while its clip is still hidden", async () => {
+ servePreview();
+ const root = mountRoot();
+ timed(root, "div", "0");
+ const soon = timed(root, "div", "1.5");
+ const later = timed(root, "div", "5");
+ const lazyPlate = (clip: HTMLElement) => {
+ const img = clip.appendChild(document.createElement("img"));
+ img.setAttribute("loading", "lazy");
+ img.setAttribute(STUDIO_PREVIEW_LAZY_ATTR, "");
+ return img;
+ };
+ const plate = lazyPlate(soon);
+ const farPlate = lazyPlate(later);
+ // A clip nested in the coming one waits for its own look-ahead.
+ const nestedPlate = lazyPlate(timed(soon, "div", "8"));
+ const authored = soon.appendChild(document.createElement("img"));
+ authored.setAttribute("loading", "lazy");
+
+ await evaluateRuntime();
+ expect(visibility(soon)).toEqual(["hidden"]);
+ expect(
+ [plate, farPlate, nestedPlate, authored].map((img) => img.getAttribute("loading")),
+ ).toEqual(["eager", "lazy", "lazy", "lazy"]);
+ window.__player?.seek(3.5);
+ expect(farPlate.getAttribute("loading")).toBe("eager");
+ });
+
it("holds a paused jump on the previous picture until the next scene's image decodes", async () => {
servePreview();
const root = mountRoot();
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 91fa173700..7b5c21fa50 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -384,6 +384,16 @@ const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";
const webAudioMediaIn = (root: ParentNode): HTMLMediaElement[] =>
Array.from(root.querySelectorAll(WEB_AUDIO_MEDIA)).filter(joinsWebAudio);
+/** Studio's lazy images in `root` load now: a hidden clip has no layout box, so lazy ones would never fetch. */
+function loadPreviewImagesIn(root: Element): void {
+ const images = root.matches(`[${STUDIO_PREVIEW_LAZY_ATTR}]`)
+ ? [root]
+ : Array.from(root.querySelectorAll(`img[${STUDIO_PREVIEW_LAZY_ATTR}]`)).filter(
+ (img) => img.closest("[data-start]") === root,
+ );
+ for (const img of images) img.setAttribute("loading", "eager");
+}
+
export function initSandboxRuntimeModular(): void {
const state = createRuntimeState();
authoredMediaObserver?.disconnect();
@@ -2798,6 +2808,8 @@ export function initSandboxRuntimeModular(): void {
rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";
const upcoming =
hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime);
+ if (upcoming && !rawNode.hasAttribute(STUDIO_PREVIEW_UPCOMING_ATTR))
+ loadPreviewImagesIn(rawNode);
rawNode.toggleAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, upcoming);
if (isMediaElement(rawNode) && metadataBoundMedia.has(rawNode) && inPreloadWindow(rawNode))
preloadNearPlayhead(
@@ -3888,7 +3900,7 @@ export function initSandboxRuntimeModular(): void {
const held = { time: quantized, apply: () => applySeek(quantized, options) };
heldSeek = held;
for (const img of undecoded) {
- if (img.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) img.setAttribute("loading", "eager");
+ loadPreviewImagesIn(img);
for (let clip = img.closest(SKIPPED_CLIP); clip; clip = img.closest(SKIPPED_CLIP))
clip.setAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, "");
}