Skip to content

Write HTML. Render video. Built for agents.

fix(cli): wait for consent navigation before cataloging assets (#4949)

Santhi Prakash0693839b5230
diff --git a/packages/cli/src/capture/coreExtractionPhase.ts b/packages/cli/src/capture/coreExtractionPhase.ts
index c9a815f6e1..b1864148fa 100644
--- a/packages/cli/src/capture/coreExtractionPhase.ts
+++ b/packages/cli/src/capture/coreExtractionPhase.ts
@@ -25,6 +25,7 @@ import {
} from "./mediaCapture.js";
import { extractVisibleText } from "./contentExtractor.js";
import { isDegradableEvaluateTimeoutError } from "./captureTimeout.js";
+import { waitForPageReady } from "./navigateForCapture.js";
import { lazyScrollForCapture } from "./lazyScrollForCapture.js";
import { filterExtractedScripts } from "./filterExtractedScripts.js";
@@ -362,6 +363,11 @@ export async function runCoreExtraction(input: CoreExtractionInput): Promise<Cor
};
await runScreenshotCapture();
const runHtmlExtraction = async (): Promise<void> => {
+ await waitForPageReady(page1, {
+ timeoutMs: Math.min(15_000, remainingMs()),
+ networkIdle: "after-load",
+ });
+
// Catalog all assets (must run before extractHtml which converts img src to data URLs)
progress("design", "Cataloging assets...");
try {
diff --git a/packages/cli/src/capture/navigateForCapture.test.ts b/packages/cli/src/capture/navigateForCapture.test.ts
index 73ba7007b9..cea2bf8ff7 100644
--- a/packages/cli/src/capture/navigateForCapture.test.ts
+++ b/packages/cli/src/capture/navigateForCapture.test.ts
@@ -6,6 +6,7 @@ import {
isNavigationTimeoutError,
navigateForCapture,
networkIdleAttemptTimeoutMs,
+ waitForPageReady,
NETWORK_IDLE_ATTEMPT_MS,
} from "./navigateForCapture.js";
@@ -120,6 +121,167 @@ describe("navigateForCapture", () => {
});
});
+describe("waitForPageReady", () => {
+ it("pays one idle wait when the document is already parsed", async () => {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const evaluate = vi.fn(async () => false);
+
+ await waitForPageReady({ waitForNetworkIdle, evaluate });
+
+ // concurrency: 2 — the networkidle2 semantic navigation already uses, so
+ // a single held-open request cannot block the wait until timeout.
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
+ expect(waitForNetworkIdle).toHaveBeenCalledWith(expect.objectContaining({ concurrency: 2 }));
+ });
+
+ it("re-waits for idle once a still-parsing document finishes loading", async () => {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ let loading = true;
+ const evaluate = vi.fn(async () => {
+ const was = loading;
+ loading = false;
+ return was;
+ });
+
+ await waitForPageReady({ waitForNetworkIdle, evaluate });
+
+ expect(evaluate).toHaveBeenCalledTimes(2);
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(2);
+ });
+
+ it("treats a destroyed execution context as mid-navigation and keeps polling", async () => {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const evaluate = vi
+ .fn()
+ .mockRejectedValueOnce(new Error("Execution context was destroyed"))
+ .mockResolvedValue(false);
+
+ await waitForPageReady({ waitForNetworkIdle, evaluate });
+
+ expect(evaluate).toHaveBeenCalledTimes(2);
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(2);
+ });
+
+ it("rethrows degradable evaluate timeouts instead of polling through them", async () => {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const evaluate = vi.fn(async () => {
+ throw new TimeoutError(
+ "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.",
+ );
+ });
+
+ await expect(waitForPageReady({ waitForNetworkIdle, evaluate })).rejects.toBeInstanceOf(
+ TimeoutError,
+ );
+ // The stage degrades on the first timeout — it is not retried as
+ // "still loading" for the full deadline.
+ expect(evaluate).toHaveBeenCalledTimes(1);
+ });
+
+ it("stays bounded by wall clock when the document never reports a parsed state", async () => {
+ vi.useFakeTimers();
+ try {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const evaluate = vi.fn(async () => true);
+
+ const start = Date.now();
+ const done = waitForPageReady({ waitForNetworkIdle, evaluate }, { timeoutMs: 15_000 });
+ await vi.runAllTimersAsync();
+ await done;
+
+ expect(Date.now() - start).toBeLessThanOrEqual(15_000);
+ // ~500ms polls over the deadline — not unbounded.
+ expect(evaluate.mock.calls.length).toBeLessThanOrEqual(31);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("bounds a hung evaluate to the shared deadline, not per-iteration latency", async () => {
+ vi.useFakeTimers();
+ try {
+ // Each probe hangs for a protocol-timeout-scale 30s; the settle must
+ // still finish at the deadline rather than paying 30s per poll.
+ const evaluate = vi.fn(
+ () =>
+ new Promise<unknown>((resolve) => {
+ setTimeout(() => resolve(true), 30_000);
+ }),
+ );
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+
+ const start = Date.now();
+ const done = waitForPageReady({ waitForNetworkIdle, evaluate }, { timeoutMs: 15_000 });
+ await vi.advanceTimersByTimeAsync(15_000);
+ await done;
+
+ expect(Date.now() - start).toBeLessThanOrEqual(15_000);
+ expect(evaluate).toHaveBeenCalledTimes(1);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("skips the leading idle wait in after-load mode on a parsed document", async () => {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const evaluate = vi.fn(async () => false);
+
+ await waitForPageReady({ waitForNetworkIdle, evaluate }, { networkIdle: "after-load" });
+
+ expect(waitForNetworkIdle).not.toHaveBeenCalled();
+ });
+
+ it("still idles in after-load mode once a loading document finishes", async () => {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ let loading = true;
+ const evaluate = vi.fn(async () => {
+ const was = loading;
+ loading = false;
+ return was;
+ });
+
+ await waitForPageReady({ waitForNetworkIdle, evaluate }, { networkIdle: "after-load" });
+
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
+ });
+
+ it("is a no-op when the deadline has already passed", async () => {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const evaluate = vi.fn(async () => false);
+
+ await waitForPageReady({ waitForNetworkIdle, evaluate }, { timeoutMs: 0 });
+
+ expect(evaluate).not.toHaveBeenCalled();
+ expect(waitForNetworkIdle).not.toHaveBeenCalled();
+ });
+
+ it("settles without waitForNetworkIdle on the page object", async () => {
+ const evaluate = vi.fn(async () => false);
+
+ await expect(waitForPageReady({ evaluate })).resolves.toBeUndefined();
+ expect(evaluate).toHaveBeenCalledTimes(1);
+ });
+
+ it("stops polling when readyState does not evaluate to a boolean", async () => {
+ vi.useFakeTimers();
+ try {
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const evaluate = vi.fn(async () => []);
+
+ const start = Date.now();
+ const done = waitForPageReady({ waitForNetworkIdle, evaluate }, { timeoutMs: 15_000 });
+ await vi.runAllTimersAsync();
+ await done;
+
+ expect(evaluate).toHaveBeenCalledTimes(1);
+ expect(Date.now() - start).toBeLessThan(15_000);
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
+
/** A page whose goto keeps Puppeteer's contract: a non-zero timeout rejects the navigation. */
function fakeChromePage() {
const session = Object.assign(new EventEmitter(), {
diff --git a/packages/cli/src/capture/navigateForCapture.ts b/packages/cli/src/capture/navigateForCapture.ts
index 29b16fa8f4..66dca7aabf 100644
--- a/packages/cli/src/capture/navigateForCapture.ts
+++ b/packages/cli/src/capture/navigateForCapture.ts
@@ -1,5 +1,9 @@
import { TimeoutError, type CDPSession, type HTTPResponse, type Page } from "puppeteer-core";
-import { isNavigationTimeoutError, withRemainingBudget } from "./captureTimeout.js";
+import {
+ isDegradableEvaluateTimeoutError,
+ isNavigationTimeoutError,
+ withRemainingBudget,
+} from "./captureTimeout.js";
export { isNavigationTimeoutError };
@@ -63,6 +67,65 @@ export async function navigateForCapture<TResponse>(
};
}
+export interface PageReadyPage {
+ waitForNetworkIdle?(options: {
+ idleTime: number;
+ timeout: number;
+ concurrency?: number;
+ }): Promise<unknown>;
+ evaluate(fn: () => unknown): Promise<unknown>;
+}
+
+export interface WaitForPageReadyOptions {
+ timeoutMs?: number;
+ networkIdle?: "always" | "after-load";
+}
+
+export async function waitForPageReady(
+ page: PageReadyPage,
+ options: WaitForPageReadyOptions = {},
+): Promise<void> {
+ const { timeoutMs = 15_000, networkIdle = "always" } = options;
+ const deadline = Date.now() + timeoutMs;
+ const remaining = () => Math.max(0, deadline - Date.now());
+ const idle = async () => {
+ try {
+ await page.waitForNetworkIdle?.({
+ idleTime: 500,
+ timeout: remaining(),
+ concurrency: 2,
+ });
+ } catch {}
+ };
+
+ if (networkIdle === "always" && remaining() > 0) await idle();
+
+ let waitedOutLoad = false;
+ while (remaining() > 0) {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const loading = await Promise.race([
+ page
+ .evaluate(() => document.readyState === "loading")
+ .catch((err: unknown) => {
+ if (isDegradableEvaluateTimeoutError(err)) throw err;
+ return true;
+ }),
+ new Promise<true>((resolve) => {
+ timer = setTimeout(() => resolve(true), Math.max(1, remaining()));
+ }),
+ ]).finally(() => {
+ if (timer !== undefined) clearTimeout(timer);
+ });
+ if (loading !== true) break;
+ waitedOutLoad = true;
+ if (remaining() > 0) {
+ await new Promise((resolve) => setTimeout(resolve, Math.min(500, remaining())));
+ }
+ }
+
+ if (waitedOutLoad && remaining() > 0) await idle();
+}
+
const PUPPETEER_LAUNCH_DEFAULT_TIMEOUT_MS = 30_000;
const PAGE_LOAD_BLOCKING_TYPES = new Set(["Document", "Script", "Stylesheet"]);
const NAMED_BLOCKING_REQUESTS = 3;
diff --git a/packages/cli/src/capture/screenshotCapture.degrade.test.ts b/packages/cli/src/capture/screenshotCapture.degrade.test.ts
index 6348434f5a..75289dac9f 100644
--- a/packages/cli/src/capture/screenshotCapture.degrade.test.ts
+++ b/packages/cli/src/capture/screenshotCapture.degrade.test.ts
@@ -17,6 +17,7 @@ describe("captureScrollScreenshots degradation", () => {
);
}),
screenshot: vi.fn(),
+ waitForNetworkIdle: vi.fn(async () => undefined),
} as unknown as Page;
const result = await captureScrollScreenshots(page, dir, { remainingMs: () => 5_000 });
diff --git a/packages/cli/src/capture/screenshotCapture.test.ts b/packages/cli/src/capture/screenshotCapture.test.ts
index cc9dbe8632..c24f93a6b5 100644
--- a/packages/cli/src/capture/screenshotCapture.test.ts
+++ b/packages/cli/src/capture/screenshotCapture.test.ts
@@ -49,7 +49,18 @@ function fakePage(
if (script === "window.innerHeight") return 1080;
});
const screenshot = vi.fn(async (_opts?: unknown) => pngBuffer(plateHeight));
- return { page: { evaluate, screenshot, ...overrides } as unknown as Page, evaluate, screenshot };
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ return {
+ page: {
+ evaluate,
+ screenshot,
+ waitForNetworkIdle,
+ ...overrides,
+ } as unknown as Page,
+ evaluate,
+ screenshot,
+ waitForNetworkIdle,
+ };
}
describe("captureFullPagePlate — the scroll shot's plate", () => {
@@ -346,7 +357,11 @@ describe("captureScrollScreenshots — capture budget", () => {
return undefined;
});
const screenshot = vi.fn(async () => pngBuffer(1080));
- const page = { evaluate, screenshot } as unknown as Page;
+ const page = {
+ evaluate,
+ screenshot,
+ waitForNetworkIdle: vi.fn(async () => undefined),
+ } as unknown as Page;
try {
const capture = captureScrollScreenshots(page, dir, {
@@ -479,6 +494,168 @@ describe("captureFullPagePlate — the guard sees the post-neutralisation page (
});
});
+describe("captureScrollScreenshots — consent-click page reload (#4923)", () => {
+ // The click-evaluate script is the one containing `el.click()`; the overlay
+ // sweep contains `createTreeWalker`. While `reloading` is set, evaluate
+ // throws like a real page mid-navigation until the idle wait settles it.
+ function settledSize(src: string) {
+ if (src.includes("scrollHeight")) return 2160;
+ if (src === "window.innerHeight") return 1080;
+ return undefined;
+ }
+ function scrollablePage(
+ evaluate: ReturnType<typeof vi.fn>,
+ waitForNetworkIdle: ReturnType<typeof vi.fn>,
+ ) {
+ const screenshot = vi.fn(async (_opts?: unknown) => pngBuffer(1080));
+ const page = { evaluate, screenshot, waitForNetworkIdle } as unknown as Page;
+ return { page, evaluate, waitForNetworkIdle };
+ }
+ function reloadingPage() {
+ let reloading = false;
+ const evaluate = vi.fn(async (script?: unknown) => {
+ const src = String(script);
+ if (src.includes("el.click()")) {
+ reloading = true;
+ return undefined;
+ }
+ if (reloading) {
+ throw new Error("Execution context was destroyed, most likely because of a navigation");
+ }
+ return settledSize(src);
+ });
+ const waitForNetworkIdle = vi.fn(async () => {
+ reloading = false;
+ });
+ return scrollablePage(evaluate, waitForNetworkIdle);
+ }
+
+ it("waits out a click-triggered reload before measuring the page", async () => {
+ const dir = tempDir("hf-scroll-reload-");
+ const { page, evaluate, waitForNetworkIdle } = reloadingPage();
+
+ const { files } = await captureScrollScreenshots(page, dir);
+
+ expect(files.length).toBeGreaterThan(0);
+ expect(waitForNetworkIdle).toHaveBeenCalled();
+ const firstIdle = waitForNetworkIdle.mock.invocationCallOrder[0]!;
+ const heightProbe = evaluate.mock.calls.findIndex((c) => String(c[0]).includes("scrollHeight"));
+ expect(heightProbe).toBeGreaterThanOrEqual(0);
+ expect(evaluate.mock.invocationCallOrder[heightProbe]!).toBeGreaterThan(firstIdle);
+ });
+
+ it("re-runs the overlay sweep on the reloaded document", async () => {
+ const dir = tempDir("hf-scroll-reload-sweep-");
+ const { page, evaluate } = reloadingPage();
+
+ await captureScrollScreenshots(page, dir);
+
+ const scripts = evaluate.mock.calls.map((c) => String(c[0]));
+ const clickIdx = scripts.findIndex((s) => s.includes("el.click()"));
+ const sweepIdx = scripts.findIndex((s) => s.includes("createTreeWalker"));
+ const heightIdx = scripts.findIndex((s) => s.includes("scrollHeight"));
+ expect(clickIdx).toBeGreaterThanOrEqual(0);
+ // Clicks, then the sweep on the settled document, then the height probe —
+ // a sweep that ran pre-reload would have been wiped with the old document.
+ expect(clickIdx).toBeLessThan(sweepIdx);
+ expect(sweepIdx).toBeLessThan(heightIdx);
+ });
+
+ it("does not stall when the clicks leave the page in place", async () => {
+ const dir = tempDir("hf-scroll-no-reload-");
+ const { page, waitForNetworkIdle } = fakePage();
+
+ const { files } = await captureScrollScreenshots(page, dir);
+
+ // The document reports parsed on the first probe, so the settle pays
+ // exactly one idle wait and the run still finishes.
+ expect(files.length).toBeGreaterThan(0);
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
+ });
+
+ it("skips the settle when nothing was clicked and the document is unchanged", async () => {
+ const dir = tempDir("hf-scroll-no-settle-");
+ // A stable timeOrigin proves the same document before and after the
+ // sweep, so a banner-free page pays the flat 400ms and no idle wait.
+ const evaluate = vi.fn(async (script?: unknown) => {
+ const src = String(script);
+ if (src.includes("performance.timeOrigin")) return 12_345;
+ if (src.includes("el.click()")) return 0;
+ return settledSize(src);
+ });
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const { page } = scrollablePage(evaluate, waitForNetworkIdle);
+
+ const { files } = await captureScrollScreenshots(page, dir);
+
+ expect(files.length).toBeGreaterThan(0);
+ expect(waitForNetworkIdle).not.toHaveBeenCalled();
+ });
+
+ it("settles when the sweep clicked a control even without a reload", async () => {
+ const dir = tempDir("hf-scroll-clicked-");
+ const evaluate = vi.fn(async (script?: unknown) => {
+ const src = String(script);
+ if (src.includes("performance.timeOrigin")) return 12_345;
+ if (src.includes("el.click()")) return 1;
+ return settledSize(src);
+ });
+ const waitForNetworkIdle = vi.fn(async () => undefined);
+ const { page } = scrollablePage(evaluate, waitForNetworkIdle);
+
+ const { files } = await captureScrollScreenshots(page, dir);
+
+ expect(files.length).toBeGreaterThan(0);
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
+ });
+
+ // Puppeteer's waitForNetworkIdle counts a request only until its response
+ // headers arrive, so it resolves while a reloaded document is still
+ // parsing. waitForNavigation cannot help there — it deadlocks when armed
+ // after the navigation commits — so the settle also polls readyState.
+ function loadingDocumentPage() {
+ let loading = false;
+ const evaluate = vi.fn(async (script?: unknown) => {
+ const src = String(script);
+ if (src.includes("el.click()")) {
+ loading = true;
+ return undefined;
+ }
+ if (src.includes("readyState")) {
+ if (!loading) return false;
+ // The document finishes parsing while the settle loop waits for it.
+ queueMicrotask(() => {
+ loading = false;
+ });
+ return true;
+ }
+ if (loading) {
+ throw new Error("Execution context was destroyed, most likely because of a navigation");
+ }
+ return settledSize(src);
+ });
+ return scrollablePage(
+ evaluate,
+ vi.fn(async () => undefined),
+ );
+ }
+
+ it("waits out a reloaded document that is still parsing after network idle", async () => {
+ const dir = tempDir("hf-scroll-loading-doc-");
+ const { page, evaluate, waitForNetworkIdle } = loadingDocumentPage();
+
+ const { files } = await captureScrollScreenshots(page, dir);
+
+ expect(files.length).toBeGreaterThan(0);
+ // A second idle wait runs once the document finishes loading, before the
+ // height probe measures the rebuilt page.
+ expect(waitForNetworkIdle).toHaveBeenCalledTimes(2);
+ const lastIdle = waitForNetworkIdle.mock.invocationCallOrder.at(-1)!;
+ const heightProbe = evaluate.mock.calls.findIndex((c) => String(c[0]).includes("scrollHeight"));
+ expect(evaluate.mock.invocationCallOrder[heightProbe]!).toBeGreaterThan(lastIdle);
+ });
+});
+
describe("captureScrollScreenshots — planted screenshots link (#4304)", () => {
// Creating symlinks needs elevated rights on Windows.
it.skipIf(process.platform === "win32")(
diff --git a/packages/cli/src/capture/screenshotCapture.ts b/packages/cli/src/capture/screenshotCapture.ts
index 8e74d4f8fe..f2a97de684 100644
--- a/packages/cli/src/capture/screenshotCapture.ts
+++ b/packages/cli/src/capture/screenshotCapture.ts
@@ -10,6 +10,7 @@ import { CaptureDirRefusedError } from "./captureErrors.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { ConnectionClosedError, type Page } from "puppeteer-core";
import { isDegradableEvaluateTimeoutError } from "./captureTimeout.js";
+import { waitForPageReady } from "./navigateForCapture.js";
import { join } from "node:path";
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
@@ -184,7 +185,15 @@ export async function captureScrollScreenshots(
// Dismiss marketing banners, cookie consents, and popups before scrolling.
// These overlay content and contaminate screenshots with UI that doesn't
// belong in video compositions (cookie popups, newsletter modals, etc.)
- await page
+ const readTimeOrigin = async (): Promise<number> => {
+ const value: unknown = await page.evaluate(`performance.timeOrigin`).catch((err: unknown) => {
+ if (isDegradableEvaluateTimeoutError(err)) throw err;
+ return undefined;
+ });
+ return typeof value === "number" ? value : Number.NaN;
+ };
+ const originBefore = await readTimeOrigin();
+ const clickedCount = await page
.evaluate(() => {
// Click common dismiss/accept buttons
const selectors = [
@@ -214,14 +223,35 @@ export async function captureScrollScreenshots(
'button[class*="not-now"]',
'button[class*="no-thanks"]',
];
+ let clicked = 0;
for (const sel of selectors) {
try {
const el = document.querySelector<HTMLElement>(sel);
- if (el) el.click();
+ if (el) {
+ el.click();
+ clicked++;
+ }
} catch {
/* ignore */
}
}
+ return clicked;
+ })
+ .catch((err: unknown) => {
+ if (isDegradableEvaluateTimeoutError(err)) {
+ throw err;
+ }
+ return 0;
+ });
+ await new Promise((r) => setTimeout(r, 400));
+ if (clickedCount > 0 || (await readTimeOrigin()) !== originBefore) {
+ await waitForPageReady(page, {
+ timeoutMs: Math.min(15_000, budget.remainingMs?.() ?? 15_000),
+ });
+ }
+ await page
+ // fallow-ignore-next-line complexity
+ .evaluate(() => {
// Hide fixed/sticky overlays that aren't the main nav. Scanning every
// element with querySelectorAll('*') + getComputedStyle is O(n) DOM
// calls and can dominate evaluate() time on large pages. Narrow the
@@ -259,7 +289,6 @@ export async function captureScrollScreenshots(
throw err;
}
});
- await new Promise((r) => setTimeout(r, 400));
const scrollHeight = (await page.evaluate(
`Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)`,