An open source design system that's fully customizable and agent ready
TypeScript
fix(ci): the CDN smoke test retries a page that never goes idle and names the request that hung (#8733)
Christopher Chedeaufcf4d7f6f919
diff --git a/.github/scripts/cdn-template-smoke-test.mjs b/.github/scripts/cdn-template-smoke-test.mjsindex 107ea5bc77..4a901bfc4c 100644--- a/.github/scripts/cdn-template-smoke-test.mjs+++ b/.github/scripts/cdn-template-smoke-test.mjs@@ -30,6 +30,11 @@ import * as os from 'node:os';import * as path from 'node:path';import {fileURLToPath} from 'node:url';import {pageAssetUrls, primeCdnAssets} from './lib/cdn-asset-priming.mjs';+import {+ renderPage,+ renderUntilClean,+ uncleanFindings,+} from './lib/cdn-page-render.mjs';import {resolvePublishedCdnVersion} from './lib/cdn-version.mjs';const ROOT = path.resolve(@@ -147,57 +152,45 @@ const server = http.createServer((req, res) => {await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));const url = `http://127.0.0.1:${server.address().port}/`;-/** Load the page once and report everything that went wrong. */-async function render() {- const consoleErrors = [];- const pageErrors = [];- const failedRequests = [];-- const browser = await chromium.launch();- try {- const page = await browser.newPage();- page.on('console', m => {- if (m.type() === 'error') consoleErrors.push(m.text().slice(0, 300));- });- page.on('pageerror', e => pageErrors.push(String(e).slice(0, 300)));- page.on('requestfailed', r =>- failedRequests.push(`${r.url()} :: ${r.failure()?.errorText}`),- );- page.on('response', r => {- if (r.status() >= 400) failedRequests.push(`${r.status()} ${r.url()}`);- });-- await page.goto(url, {waitUntil: 'networkidle', timeout: 90_000});- // A page that throws during render still reaches networkidle with an empty- // root, so the render itself has to be asserted, not just the error streams.- const button = await page- .getByRole('button')- .first()- .textContent({timeout: 15_000})- .catch(() => null);-- return {consoleErrors, pageErrors, failedRequests, button};- } finally {- await browser.close();- }-}-// ── 5. Render, retrying once so a blip at a CDN is not a red build ───────────-let result;-for (let attempt = 1; attempt <= ATTEMPTS; attempt++) {- result = await render();- const clean =- result.consoleErrors.length === 0 &&- result.pageErrors.length === 0 &&- result.failedRequests.length === 0 &&- result.button;- if (clean) break;- if (attempt < ATTEMPTS) console.log(` retry ${attempt} of ${ATTEMPTS - 1}`);-}+// Every attempt shares the one 90 s wait this step has always had, so the retry+// adds no time to it: this step can start with under two minutes of the+// smoke-test job's 15-minute limit left, and a page that hangs on both+// attempts still prints the report naming what hung before the job is stopped.+const RENDER_BUDGET_MS = 90_000;+const result = await renderUntilClean(+ ({budgetMs}) =>+ renderPage(url, {launch: async () => chromium.launch(), budgetMs}),+ {+ attempts: ATTEMPTS,+ deadline: Date.now() + RENDER_BUDGET_MS,+ // The attempt a retry replaces is printed in full: a CDN blip that clears+ // on the retry still shows in the log, rather than vanishing behind a+ // green run.+ onRetry: (attempt, leftMs, discarded) => {+ console.log(+ ` retry ${attempt} of ${ATTEMPTS - 1}, ${Math.round(leftMs / 1000)} s left, after:`,+ );+ for (const finding of uncleanFindings(discarded)) {+ console.log(` ${finding}`);+ }+ },+ },+);server.close();// ── 6. Report ────────────────────────────────────────────────────────────────console.log(`\n${PAGE} @ ${rendered}`);+if (result.timedOut) {+ fail(+ 'the page never went idle, so its navigation timed out',+ result.pendingRequests.length === 0+ ? ' no request was in flight when the wait gave up'+ : result.pendingRequests+ .map(entry => ` still in flight: ${entry}`)+ .join('\n'),+ );+}for (const [label, entries] of [['console error', result.consoleErrors],['page error', result.pageErrors],@@ -214,7 +207,7 @@ for (const [label, entries] of [}if (result.button) {console.log(` ok rendered a button: "${result.button.trim()}"`);-} else {+} else if (!result.timedOut) {fail('the page loaded but rendered no button');}diff --git a/.github/scripts/lib/cdn-page-render.browser.test.mjs b/.github/scripts/lib/cdn-page-render.browser.test.mjsnew file mode 100644index 0000000000..8aa11b1bd3--- /dev/null+++ b/.github/scripts/lib/cdn-page-render.browser.test.mjs@@ -0,0 +1,81 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file cdn-page-render.browser.test.mjs+ * @input A local page whose module request is never answered, and Chromium;+ * run with ASTRYX_CDN_SMOKE_BROWSER=1 after `npx playwright install chromium`+ * @output Real-browser proof that a page with a request that hangs never goes+ * idle, and that renderPage reports that navigation timeout with the hung+ * request instead of throwing+ * @position The browser half of cdn-page-render.test.mjs, which runs the same+ * rules against Playwright's events without a browser+ */++import http from 'node:http';+import {chromium} from 'playwright';+import {afterAll, beforeAll, describe, expect, it} from 'vitest';+import {renderPage, renderUntilClean} from './cdn-page-render.mjs';++const PAGE =+ '<!doctype html><button>Start</button>' ++ '<script type="module" src="/stall.js"></script>';++describe.runIf(process.env.ASTRYX_CDN_SMOKE_BROWSER === '1')(+ 'renderPage in Chromium',+ () => {+ /** The module responses held open, so the page can never go idle. */+ const held = [];+ let server;+ let url;++ beforeAll(async () => {+ server = http.createServer((req, res) => {+ if (req.url === '/stall.js') {+ held.push(res);+ return;+ }+ res.writeHead(200, {'content-type': 'text/html; charset=utf-8'});+ res.end(PAGE);+ });+ await new Promise(resolve => {+ server.listen(0, '127.0.0.1', resolve);+ });+ url = `http://127.0.0.1:${server.address().port}/`;+ });++ afterAll(async () => {+ for (const res of held) {+ res.destroy();+ }+ server.closeAllConnections();+ await new Promise(resolve => {+ server.close(resolve);+ });+ });++ it('reports a page that never goes idle, naming the module that hung', async () => {+ const result = await renderPage(url, {+ launch: async () => chromium.launch(),+ navigationTimeoutMs: 2_000,+ });+ expect(result.timedOut).toBe(true);+ expect(result.pendingRequests).toEqual([`${url}stall.js`]);+ });++ it('fits both attempts at a page that never goes idle inside one deadline', async () => {+ const started = Date.now();+ const result = await renderUntilClean(+ (attempt = {}) =>+ renderPage(url, {+ launch: async () => chromium.launch(),+ budgetMs: attempt.budgetMs,+ }),+ {attempts: 2, deadline: Date.now() + 4_000},+ );+ // Two browser launches beside the 4 s the attempts share.+ expect(Date.now() - started).toBeLessThan(8_000);+ expect(result.timedOut).toBe(true);+ expect(result.pendingRequests).toEqual([`${url}stall.js`]);+ }, 20_000);+ },+);diff --git a/.github/scripts/lib/cdn-page-render.mjs b/.github/scripts/lib/cdn-page-render.mjsnew file mode 100644index 0000000000..186173d479--- /dev/null+++ b/.github/scripts/lib/cdn-page-render.mjs@@ -0,0 +1,196 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * Render the CDN starter page in a browser, and report what went wrong.+ *+ * One render loads the page until its network goes idle, then reads the first+ * button: a page that throws while rendering still goes idle with an empty+ * root, so the render itself is asserted, not only the error streams. A page+ * with a request that never finishes never goes idle, so its navigation times+ * out: that is an unclean render like any other, reported with the requests+ * still in flight when the wait gave up, so the output names the asset that+ * hung instead of ending the run on a stack trace. renderUntilClean retries+ * an unclean render once, so a blip at a CDN is not a red build, and fits+ * every attempt inside one deadline, so a page that hangs on each attempt+ * still gets its report printed before the job running it is stopped. It+ * hands every retry the attempt it discards, and uncleanFindings names what+ * made that attempt unclean, so a blip that clears on the retry still shows+ * in the log. Only an expected absence (a wait that ran out) becomes a value;+ * every other error throws.+ */++/** Playwright's error for a navigation whose wait condition never came. */+function isTimeout(error) {+ return error?.name === 'TimeoutError';+}++/**+ * Load `url` once in a browser from `launch`, and collect its console errors,+ * page errors and failed requests, and the first button's text (null when no+ * button rendered). A navigation that times out comes back with `timedOut`+ * and the URLs still in flight (`pendingRequests`, in request order); any+ * other navigation error throws. With `budgetMs`, the whole attempt fits in+ * it: the navigation waits no longer than the budget, and the button no+ * longer than what is left of it.+ *+ * @param {string} url+ * @param {{+ * launch: () => Promise<import('playwright').Browser>,+ * navigationTimeoutMs?: number,+ * buttonTimeoutMs?: number,+ * budgetMs?: number,+ * now?: () => number,+ * }} options+ */+export async function renderPage(+ url,+ {+ launch,+ navigationTimeoutMs = 90_000,+ buttonTimeoutMs = 15_000,+ budgetMs,+ now = Date.now,+ },+) {+ const startedAt = now();+ /**+ * `limitMs`, cut to what is left of the budget. Never 0: Playwright reads a+ * timeout of 0 as no timeout at all.+ */+ const withinBudget = limitMs =>+ budgetMs == null+ ? limitMs+ : Math.max(1, Math.min(limitMs, budgetMs - (now() - startedAt)));+ const consoleErrors = [];+ const pageErrors = [];+ const failedRequests = [];+ /** Requests the page made that have neither finished nor failed. */+ const inFlight = new Set();++ const browser = await launch();+ try {+ const page = await browser.newPage();+ page.on('console', m => {+ if (m.type() === 'error') consoleErrors.push(m.text().slice(0, 300));+ });+ page.on('pageerror', e => pageErrors.push(String(e).slice(0, 300)));+ page.on('request', r => inFlight.add(r));+ page.on('requestfinished', r => inFlight.delete(r));+ page.on('requestfailed', r => {+ inFlight.delete(r);+ failedRequests.push(`${r.url()} :: ${r.failure()?.errorText}`);+ });+ page.on('response', r => {+ if (r.status() >= 400) failedRequests.push(`${r.status()} ${r.url()}`);+ });++ try {+ await page.goto(url, {+ waitUntil: 'networkidle',+ timeout: withinBudget(navigationTimeoutMs),+ });+ } catch (error) {+ if (!isTimeout(error)) {+ throw error;+ }+ return {+ consoleErrors,+ pageErrors,+ failedRequests,+ timedOut: true,+ pendingRequests: [...inFlight].map(r => r.url()),+ button: null,+ };+ }+ let button;+ try {+ button = await page+ .getByRole('button')+ .first()+ .textContent({timeout: withinBudget(buttonTimeoutMs)});+ } catch (error) {+ if (!isTimeout(error)) {+ throw error;+ }+ // No button within the wait: the page rendered nothing to press.+ button = null;+ }++ return {+ consoleErrors,+ pageErrors,+ failedRequests,+ timedOut: false,+ pendingRequests: [],+ button,+ };+ } finally {+ await browser.close();+ }+}++/**+ * Everything that made `result` unclean, one line each: the timeout and the+ * requests still in flight, each console error, page error and failed+ * request, and a missing button on a page that finished loading. Empty for a+ * clean render.+ */+export function uncleanFindings(result) {+ return [+ ...(result.timedOut ? ['navigation timed out'] : []),+ ...result.pendingRequests.map(url => `still in flight: ${url}`),+ ...result.consoleErrors.map(entry => `console error: ${entry}`),+ ...result.pageErrors.map(entry => `page error: ${entry}`),+ ...result.failedRequests.map(entry => `failed request: ${entry}`),+ ...(!result.timedOut && !result.button ? ['rendered no button'] : []),+ ];+}++/** Whether a render went idle with nothing wrong and a button on the page. */+export function isCleanRender(result) {+ return (+ !result.timedOut &&+ result.consoleErrors.length === 0 &&+ result.pageErrors.length === 0 &&+ result.failedRequests.length === 0 &&+ Boolean(result.button)+ );+}++/**+ * Run `render` until one comes back clean, at most `attempts` times, and+ * return the last result. With a `deadline` (in `now()`'s clock), each attempt+ * is handed `budgetMs`, an equal share of the time left (so the retry gets+ * whatever the first attempt left unused), and no attempt starts once the+ * deadline has passed. `onRetry(attempt, leftMs, discarded)` hears each+ * retry, with the unclean attempt it replaces.+ *+ * @template {{timedOut: boolean, consoleErrors: string[], pageErrors: string[], failedRequests: string[], button: string | null}} R+ * @param {(attempt: {budgetMs?: number}) => Promise<R>} render+ * @param {{+ * attempts?: number,+ * deadline?: number,+ * now?: () => number,+ * onRetry?: (attempt: number, leftMs: number | undefined, discarded: R) => void,+ * }} [options]+ * @returns {Promise<R>}+ */+export async function renderUntilClean(+ render,+ {attempts = 2, deadline, now = Date.now, onRetry} = {},+) {+ const leftMs = () => (deadline == null ? undefined : deadline - now());+ let result;+ for (let attempt = 1; attempt <= attempts; attempt++) {+ const left = leftMs();+ result = await render({+ budgetMs:+ left == null ? undefined : Math.floor(left / (attempts - attempt + 1)),+ });+ if (isCleanRender(result) || attempt === attempts) break;+ const remaining = leftMs();+ if (remaining != null && remaining <= 0) break;+ onRetry?.(attempt, remaining, result);+ }+ return result;+}diff --git a/.github/scripts/lib/cdn-page-render.test.mjs b/.github/scripts/lib/cdn-page-render.test.mjsnew file mode 100644index 0000000000..f67e7091ef--- /dev/null+++ b/.github/scripts/lib/cdn-page-render.test.mjs@@ -0,0 +1,346 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {EventEmitter} from 'node:events';+import {describe, expect, it} from 'vitest';+import {+ isCleanRender,+ renderPage,+ renderUntilClean,+ uncleanFindings,+} from './cdn-page-render.mjs';++const URL = 'http://127.0.0.1:1/';+const CORE = 'https://esm.sh/@astryxdesign/[email protected]?external=react,react-dom';+const CSS =+ 'https://cdn.jsdelivr.net/npm/@astryxdesign/[email protected]/dist/astryx.css';++/** A request as Playwright's page events carry it. */+function request(url, errorText = null) {+ return {+ url: () => url,+ failure: () => (errorText == null ? null : {errorText}),+ };+}++/** The error Playwright's `page.goto` throws when its wait never completes. */+function navigationTimeout() {+ const error = new Error('page.goto: Timeout 90000ms exceeded.');+ error.name = 'TimeoutError';+ return error;+}++/**+ * A browser `launch` whose pages run `onGoto(page, launchIndex, options)`+ * while navigating, then settle the navigation with `gotoErrors[launchIndex]`+ * (none resolves it), and answer the button query with `button` after running+ * `onButton(options)`.+ */+function fakeLaunch({+ onGoto = () => {},+ onButton = () => {},+ gotoErrors = [],+ button = 'Start',+ buttonError = null,+}) {+ let launches = 0;+ let closed = 0;+ const launch = async () => {+ const index = launches;+ launches += 1;+ const page = new EventEmitter();+ page.goto = async (_url, options) => {+ onGoto(page, index, options);+ const error = gotoErrors[index];+ if (error != null) {+ throw error;+ }+ };+ page.getByRole = () => ({+ first: () => ({+ textContent: async options => {+ onButton(options);+ if (buttonError != null) {+ throw buttonError;+ }+ return button;+ },+ }),+ });+ return {+ newPage: async () => page,+ close: async () => {+ closed += 1;+ },+ };+ };+ return {launch, launches: () => launches, closed: () => closed};+}++/** A render whose navigation timed out with the core module still in flight. */+const TIMED_OUT = {+ consoleErrors: [],+ pageErrors: [],+ failedRequests: [],+ timedOut: true,+ pendingRequests: [CORE],+ button: null,+};++describe('renderPage', () => {+ it('reports a page that goes idle and renders a button as clean', async () => {+ const {launch, closed} = fakeLaunch({+ onGoto: page => {+ const css = request(CSS);+ page.emit('request', css);+ page.emit('requestfinished', css);+ },+ });+ const result = await renderPage(URL, {launch});+ expect(result.button).toBe('Start');+ expect(isCleanRender(result)).toBe(true);+ expect(closed()).toBe(1);+ });++ it('reports a navigation that times out as an unclean render, naming the requests still in flight', async () => {+ const {launch, closed} = fakeLaunch({+ onGoto: page => {+ const css = request(CSS);+ const core = request(CORE);+ page.emit('request', css);+ page.emit('requestfinished', css);+ page.emit('request', core);+ },+ gotoErrors: [navigationTimeout()],+ });+ const result = await renderPage(URL, {launch});+ expect(result.timedOut).toBe(true);+ expect(result.pendingRequests).toEqual([CORE]);+ expect(isCleanRender(result)).toBe(false);+ expect(closed()).toBe(1);+ });++ it('counts a failed request as failed, not as still in flight', async () => {+ const {launch} = fakeLaunch({+ onGoto: page => {+ const core = request(CORE, 'net::ERR_BLOCKED_BY_ORB');+ page.emit('request', core);+ page.emit('requestfailed', core);+ },+ gotoErrors: [navigationTimeout()],+ });+ const result = await renderPage(URL, {launch});+ expect(result.failedRequests).toEqual([+ `${CORE} :: net::ERR_BLOCKED_BY_ORB`,+ ]);+ expect(result.pendingRequests).toEqual([]);+ });++ it('still throws a navigation error that is not a timeout: the local server failing is not a CDN blip', async () => {+ const refused = new Error('page.goto: net::ERR_CONNECTION_REFUSED');+ const {launch, closed} = fakeLaunch({gotoErrors: [refused]});+ await expect(renderPage(URL, {launch})).rejects.toBe(refused);+ expect(closed()).toBe(1);+ });+});++describe('renderPage reading the button', () => {+ it('reports no button when none appears within the wait: the page rendered nothing to press', async () => {+ const timeout = navigationTimeout();+ const {launch} = fakeLaunch({buttonError: timeout});+ const result = await renderPage(URL, {launch});+ expect(result.button).toBeNull();+ expect(isCleanRender(result)).toBe(false);+ });++ it('throws any other error from reading the button, rather than reporting it as a missing button', async () => {+ const crashed = new Error('locator.textContent: Target crashed');+ const {launch, closed} = fakeLaunch({buttonError: crashed});+ await expect(renderPage(URL, {launch})).rejects.toBe(crashed);+ expect(closed()).toBe(1);+ });+});++describe('uncleanFindings', () => {+ it('names everything that made a render unclean', () => {+ expect(+ uncleanFindings({+ consoleErrors: ['Uncaught TypeError: x is not a function'],+ pageErrors: ['Error: boom'],+ failedRequests: [`${CSS} :: net::ERR_BLOCKED_BY_ORB`],+ timedOut: false,+ pendingRequests: [],+ button: null,+ }),+ ).toEqual([+ 'console error: Uncaught TypeError: x is not a function',+ 'page error: Error: boom',+ `failed request: ${CSS} :: net::ERR_BLOCKED_BY_ORB`,+ 'rendered no button',+ ]);+ });++ it('names a timeout with each request still in flight, and no missing button for a page that never finished', () => {+ expect(uncleanFindings(TIMED_OUT)).toEqual([+ 'navigation timed out',+ `still in flight: ${CORE}`,+ ]);+ });++ it('has nothing to say about a clean render', () => {+ expect(+ uncleanFindings({+ consoleErrors: [],+ pageErrors: [],+ failedRequests: [],+ timedOut: false,+ pendingRequests: [],+ button: 'Start',+ }),+ ).toEqual([]);+ });+});++describe('renderPage within a budget', () => {+ it('waits for the navigation no longer than its budget, and for the button no longer than what is left of it', async () => {+ let clock = 0;+ const timeouts = {};+ const {launch} = fakeLaunch({+ onGoto: (_page, _index, options) => {+ timeouts.navigation = options.timeout;+ clock += 30_000;+ },+ onButton: options => {+ timeouts.button = options.timeout;+ },+ });+ await renderPage(URL, {launch, budgetMs: 40_000, now: () => clock});+ expect(timeouts).toEqual({navigation: 40_000, button: 10_000});+ });++ it('never asks for a timeout of 0, which Playwright reads as no timeout at all', async () => {+ let clock = 0;+ const timeouts = {};+ const {launch} = fakeLaunch({+ onGoto: (_page, _index, options) => {+ timeouts.navigation = options.timeout;+ clock += 5_000;+ },+ onButton: options => {+ timeouts.button = options.timeout;+ },+ });+ await renderPage(URL, {launch, budgetMs: 5_000, now: () => clock});+ expect(timeouts.navigation).toBe(5_000);+ expect(timeouts.button).toBeGreaterThan(0);+ });+});++describe('renderUntilClean', () => {+ it('retries a navigation timeout once, then reports the requests still in flight', async () => {+ const {launch, launches} = fakeLaunch({+ onGoto: page => page.emit('request', request(CORE)),+ gotoErrors: [navigationTimeout(), navigationTimeout()],+ });+ const retries = [];+ const result = await renderUntilClean(() => renderPage(URL, {launch}), {+ attempts: 2,+ onRetry: attempt => retries.push(attempt),+ });+ expect(launches()).toBe(2);+ expect(retries).toEqual([1]);+ expect(result.timedOut).toBe(true);+ expect(result.pendingRequests).toEqual([CORE]);+ });++ it('hands each retry the attempt it discards, so what went wrong on it can be reported', async () => {+ const heard = [];+ let calls = 0;+ const clean = {+ ...TIMED_OUT,+ timedOut: false,+ pendingRequests: [],+ button: 'Start',+ };+ const result = await renderUntilClean(+ async () => {+ calls += 1;+ return calls === 1 ? TIMED_OUT : clean;+ },+ {+ attempts: 2,+ onRetry: (attempt, _leftMs, discarded) =>+ heard.push([attempt, discarded]),+ },+ );+ expect(heard).toEqual([[1, TIMED_OUT]]);+ expect(result).toBe(clean);+ });++ it('gives each attempt an equal share of the time left before the deadline', async () => {+ let clock = 0;+ const budgets = [];+ await renderUntilClean(+ async ({budgetMs}) => {+ budgets.push(budgetMs);+ clock += budgetMs;+ return TIMED_OUT;+ },+ {attempts: 2, deadline: 90_000, now: () => clock},+ );+ expect(budgets).toEqual([45_000, 45_000]);+ });++ it('gives the retry what the first attempt left unused', async () => {+ let clock = 0;+ const budgets = [];+ await renderUntilClean(+ async ({budgetMs}) => {+ budgets.push(budgetMs);+ clock += 10_000;+ return {...TIMED_OUT, timedOut: false, failedRequests: [CORE]};+ },+ {attempts: 2, deadline: 90_000, now: () => clock},+ );+ expect(budgets).toEqual([45_000, 80_000]);+ });++ it('does not retry once the deadline has passed, and reports the attempt it has', async () => {+ let clock = 0;+ let calls = 0;+ const retries = [];+ const result = await renderUntilClean(+ async () => {+ calls += 1;+ clock += 95_000;+ return TIMED_OUT;+ },+ {+ attempts: 2,+ deadline: 90_000,+ now: () => clock,+ onRetry: attempt => retries.push(attempt),+ },+ );+ expect(calls).toBe(1);+ expect(retries).toEqual([]);+ expect(result).toBe(TIMED_OUT);+ });++ it('stops at the first clean render after a timeout', async () => {+ const {launch, launches} = fakeLaunch({+ onGoto: (page, index) => {+ const core = request(CORE);+ page.emit('request', core);+ if (index > 0) {+ page.emit('requestfinished', core);+ }+ },+ gotoErrors: [navigationTimeout()],+ });+ const result = await renderUntilClean(() => renderPage(URL, {launch}), {+ attempts: 2,+ });+ expect(launches()).toBe(2);+ expect(isCleanRender(result)).toBe(true);+ });+});