Skip to content

armgit / fx0

Public Imported from GitHub
0

A lightweight design agent built with libfx. Describe a page and an agent writes it as one HTML file.

TypeScript

Add a real-network first-preview A/B harness

Esteban Suarez7698ca8e8097
diff --git a/tests/helpers/preview-pane-fixture.mjs b/tests/helpers/preview-pane-fixture.mjs
new file mode 100644
index 0000000..efe3809
--- /dev/null
+++ b/tests/helpers/preview-pane-fixture.mjs
@@ -0,0 +1,124 @@
+import assert from 'node:assert/strict'
+import { execFile } from 'node:child_process'
+import { readFile, realpath, writeFile } from 'node:fs/promises'
+import { registerHooks } from 'node:module'
+import { join } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { promisify } from 'node:util'
+
+// Shared by preview-update.browser.mjs and the live browser runner. This module registers no tests.
+
+// Restore the old JSON-first behavior only in a temporary baseline compile. The candidate
+// runs the current production component unchanged; no benchmark switch enters production.
+export const LEGACY_BASELINE = {
+ marker: 'fx0-perf-baseline-json-first',
+ from: 'if (!shown) {\n loadFull(desired);\n return;\n }',
+ to: '/* fx0-perf-baseline-json-first */',
+}
+
+// Observe real window messages without intercepting any production listener or send.
+export const OBSERVER = `
+window.observed = { paints: [], results: [], boots: [], reports: [], errors: [], frameErrors: [] }
+addEventListener('message', event => {
+ const frames = [...document.querySelectorAll('#active, #preview-pane iframe')], data = event.data
+ if (!data || typeof data !== 'object') return
+ const wrapper = frames.find(frame => event.source === frame.contentWindow)
+ if (wrapper && event.origin === location.origin) {
+ const entry = { ...data, mountUrl: wrapper.getAttribute('src'), observedAt: performance.now() }
+ if (data.type === 'fx0-artifact-painted') observed.paints.push(entry)
+ if (data.type === 'fx0-preview-result') observed.results.push(entry)
+ if (data.type === 'fx0-preview-error') observed.frameErrors.push(entry)
+ }
+ const frame = frames.find(frame => event.source === frame.contentWindow.frames[0])
+ if (frame && data.fixture === 'preview-update') {
+ const entry = { ...data, mountUrl: frame.getAttribute('src') }
+ if (data.kind === 'boot') observed.boots.push(entry)
+ if (data.kind === 'state') { observed.reports.push(entry); if (observed.reports.length > 30) observed.reports.shift() }
+ if (data.kind === 'error') observed.errors.push(data.error)
+ }
+})`
+
+export async function buildPaneFixture(directory, experiment = null, { sessionId = 'preview-browser-session', initialRevision = 101 } = {}) {
+ assert.ok(Number.isSafeInteger(initialRevision) && initialRevision >= 0, 'initialRevision is a non-negative integer')
+ const output = await realpath(directory), root = fileURLToPath(new URL('../../', import.meta.url))
+ // Same narrow compile as html-ui.test.mjs, using this test's Node 24 executable.
+ await promisify(execFile)(process.execPath, [join(root, 'node_modules/typescript/bin/tsc'),
+ '--ignoreConfig', '--outDir', join(output, 'compiled'), '--strict', '--types', 'node',
+ '--jsx', 'react-jsx', '--target', 'esnext', '--module', 'esnext',
+ '--moduleResolution', 'bundler', '--skipLibCheck', '--esModuleInterop', 'components/preview-pane.tsx',
+ ], { cwd: root, maxBuffer: 4 * 1024 * 1024 })
+ if (experiment) {
+ // Perf-only: remove the direct-load guard in the temporary legacy baseline, never in production.
+ const file = join(output, 'compiled/components/preview-pane.js'), source = await readFile(file, 'utf8')
+ assert.equal(source.split(experiment.from).length - 1, 1, 'the baseline matches exactly one compiled guard')
+ await writeFile(file, source.replace(experiment.from, experiment.to))
+ assert.equal((await readFile(file, 'utf8')).split(experiment.marker).length - 1, 1, 'the baseline marker is unique')
+ }
+ await writeFile(join(output, 'package.json'), '{"type":"module"}')
+ // These controls change only the real component's props. No replacement hooks,
+ // updater, transport or slot coordinator is bundled into the fixture.
+ await writeFile(join(output, 'pane-fixture.js'), `
+import { createElement, useEffect, useState } from 'react'
+import PreviewPane from './compiled/components/preview-pane.js'
+import { signalAuthRequired } from './compiled/components/auth-ui.js'
+import { captureFrame } from './compiled/components/design-capture.js'
+export function Fixture() {
+ const [revision, setRevision] = useState(${JSON.stringify(initialRevision)}), [viewing, setViewing] = useState(null)
+ const [sessionId, setSession] = useState(${JSON.stringify(sessionId)}), [draft, setDraft] = useState(null)
+ const [accessAllowed, setAccessAllowed] = useState(true), [capture, setCapture] = useState(null)
+ useEffect(() => {
+ window.captureResults = []
+ window.authErrors = []
+ Object.assign(window, { setRevision, setViewing, setSession, setDraft, setAccessAllowed, setCapture, captureFrame, closeAuthGate: signalAuthRequired, fixtureReady: true })
+ const revoke = event => { window.authErrors.push(event.detail); setAccessAllowed(false); setRevision(0) }
+ window.addEventListener('fx0:auth-required', revoke)
+ return () => window.removeEventListener('fx0:auth-required', revoke)
+ }, [])
+ return createElement(PreviewPane, { source: { kind: 'design', sessionId, revision }, viewing, draft, accessAllowed, intro: false, onClose: () => setRevision(0),
+ capture, onCaptureDone: (cid, answered) => { window.captureResults.push({ cid, answered, observedAt: performance.now() }); setCapture(current => current?.cid === cid ? null : current) },
+ })
+}`)
+ await writeFile(join(output, 'pane-entry.js'), `
+import { createElement } from 'react'
+import { hydrateRoot } from 'react-dom/client'
+import { Fixture } from './pane-fixture.js'
+const root = hydrateRoot(document.getElementById('app'), createElement(Fixture), { onRecoverableError: error => observed.errors.push(error.message) })
+window.unmountFixture = () => root.unmount()
+`)
+ const { default: bundledWebpack } = await import('../../node_modules/next/dist/compiled/webpack/webpack.js')
+ const compiler = bundledWebpack.webpack({
+ // Next's packaged webpack references a terser entry absent in this install.
+ // Production React is still selected; minification is not under test here.
+ mode: 'production', optimization: { minimize: false }, target: 'web', entry: join(output, 'pane-entry.js'),
+ output: { path: join(output, 'browser'), filename: 'bundle.js' },
+ module: { rules: [{ test: /\.js$/, resolve: { fullySpecified: false } }] },
+ resolve: { modules: [join(root, 'node_modules')], extensions: ['.js'], extensionAlias: { '.ts': ['.js'] } },
+ })
+ try {
+ const stats = await promisify(compiler.run.bind(compiler))()
+ assert.equal(stats.hasErrors(), false, stats.toString({ all: false, errors: true }))
+ } finally { await promisify(compiler.close.bind(compiler))() }
+
+ const outputUrl = pathToFileURL(`${output}/`).href, projectUrl = new URL('../../package.json', import.meta.url).href
+ const hooks = registerHooks({ resolve(specifier, context, nextResolve) {
+ if (!context.parentURL?.startsWith(outputUrl)) return nextResolve(specifier, context)
+ if (specifier.startsWith('.')) return nextResolve(/\.js$/.test(specifier) ? specifier : `${specifier.replace(/\.ts$/, '')}.js`, context)
+ return nextResolve(specifier, { ...context, parentURL: projectUrl })
+ } })
+ let markup
+ try {
+ const [{ Fixture }, { createElement }, { renderToString }] = await Promise.all([
+ import(pathToFileURL(join(output, 'pane-fixture.js')).href), import('react'), import('react-dom/server'),
+ ])
+ markup = renderToString(createElement(Fixture))
+ } finally { hooks.deregister() }
+ return {
+ bundle: await readFile(join(output, 'browser/bundle.js')),
+ html: `<!doctype html><html><head><meta charset="utf-8"><title>Real PreviewPane</title><style>
+html,body,#app{margin:0;height:100%}#preview-pane{height:100%;display:flex;flex-direction:column}
+#preview-pane>div:last-of-type{position:relative;flex:1;min-height:720px}
+#preview-pane iframe{position:absolute;inset:0;width:1000px;height:720px;border:0}
+#preview-pane iframe[aria-hidden=true]{z-index:0;pointer-events:none}#preview-pane iframe:not([aria-hidden=true]){z-index:1}
+</style></head><body><script>${OBSERVER}</script><div id="app">${markup}</div><script defer src="/bundle.js"></script></body></html>`,
+ }
+}
diff --git a/tests/preview-live-ab.mjs b/tests/preview-live-ab.mjs
new file mode 100644
index 0000000..98c52d7
--- /dev/null
+++ b/tests/preview-live-ab.mjs
@@ -0,0 +1,504 @@
+// Live A/B for the first design preview load, in one owned tab of the normal, authenticated Chrome.
+// A reconstructs the old JSON-first client flow: the helper's LEGACY_BASELINE guard is removed from the
+// compiled PreviewPane. It is not the deployed old bundle. B is the unmodified current component.
+// Both run in the same tab against the real /api/artifact-frame with the user's own cookies; nothing is
+// mocked, delayed or retried. Only read requests are made, plus client-side revision changes and captures.
+// Warm policy: the ordinary HTTP cache stays on; 2 warmup pairs are excluded; each sample is bracketed by a
+// read-only source probe. Clocks are performance.now in the page; the host only polls over AppleEvents.
+// imageReady = setRevision to the encoded image Blob. Decoding validates it afterward, outside the clock.
+// It excludes server publication, LLM and upload work.
+// Limits: one tab, network path and revision; the reused Next page's own timers may keep running after
+// document.open; the baseline is a reconstruction; 30 pairs is a small sample for the bootstrap interval.
+import { execFile } from 'node:child_process'
+import { existsSync } from 'node:fs'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { randomBytes } from 'node:crypto'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { MAX_CAPTURE_BYTES, MAX_CAPTURE_TILES } from '../lib/capture-protocol.ts'
+import { LEGACY_BASELINE, OBSERVER, buildPaneFixture } from './helpers/preview-pane-fixture.mjs'
+
+const execFileAsync = promisify(execFile)
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+const FRAME = { width: 1000, height: 720 } // the fixture's front iframe size (helper CSS)
+const IMAGE_TYPES = ['image/webp', 'image/jpeg', 'image/png']
+const WARMUP_ORDERS = ['AB', 'BA']
+const BOOTSTRAP_RESAMPLES = 2000
+const TAB_TIMEOUT_MS = 30_000
+const USAGE = 'usage: node --experimental-strip-types tests/preview-live-ab.mjs --tab ID --session UUID --out FILE [--window ID] [--pairs N (even, 2..200)] [--seed UINT32]'
+
+// Picks the tab by its id as text, activates it, and runs the JS payload that the host wrote to a temp file.
+const TAB_SCRIPT = `on run argv
+ set tabId to item 1 of argv
+ set requiredWindow to item 3 of argv
+ set jsText to read (POSIX file (item 2 of argv)) as «class utf8»
+ try
+ tell application "Google Chrome"
+ repeat with w in windows
+ repeat with i from 1 to count of tabs of w
+ set t to tab i of w
+ if ((id of t) as text) is tabId then
+ if requiredWindow is not "" and ((id of w) as text) is not requiredWindow then return "ERROR:The owned tab moved to another window"
+ set bounds of w to {0, 24, 1440, 1000}
+ set active tab index of w to i
+ activate
+ return "WINDOW:" & ((id of w) as text) & "|" & (execute t javascript jsText)
+ end if
+ end repeat
+ end repeat
+ end tell
+ return "NOTFOUND"
+ on error errMessage number errNumber
+ return "ERROR:" & errNumber & ":" & errMessage
+ end try
+end run
+`
+
+export function parseArgs(argv) {
+ const options = { tab: null, window: null, session: null, pairs: 30, seed: null, out: null }
+ for (let index = 0; index < argv.length; index += 2) {
+ const flag = argv[index], value = argv[index + 1]
+ if (value === undefined || value.startsWith('--')) throw new Error(`missing value for ${flag}. ${USAGE}`)
+ if (flag === '--tab') options.tab = value
+ else if (flag === '--window') options.window = value
+ else if (flag === '--session') options.session = value
+ else if (flag === '--pairs') options.pairs = Number(value)
+ else if (flag === '--seed') options.seed = Number(value)
+ else if (flag === '--out') options.out = value
+ else throw new Error(`unknown option ${flag}. ${USAGE}`)
+ }
+ if (!/^\d+$/.test(options.tab ?? '')) throw new Error(`--tab must be the numeric tab id. ${USAGE}`)
+ if (options.window !== null && !/^\d+$/.test(options.window)) throw new Error('--window must be a numeric window id')
+ if (!UUID.test(options.session ?? '')) throw new Error(`--session must be the preview session UUID. ${USAGE}`)
+ if (!options.out) throw new Error(`--out is required. ${USAGE}`)
+ if (!Number.isSafeInteger(options.pairs) || options.pairs < 2 || options.pairs > 200 || options.pairs % 2) {
+ throw new Error('--pairs must be an even integer from 2 to 200, so AB and BA counts match')
+ }
+ options.seed ??= randomBytes(4).readUInt32BE()
+ if (!Number.isInteger(options.seed) || options.seed < 0 || options.seed > 0xffffffff) throw new Error('--seed must be a uint32')
+ return options
+}
+
+// mulberry32: a small seeded PRNG, so the schedule and the bootstrap are reproducible from the output seed.
+export function mulberry32(seed) {
+ let state = seed >>> 0
+ return () => {
+ state = (state + 0x6d2b79f5) >>> 0
+ let t = state
+ t = Math.imul(t ^ (t >>> 15), t | 1)
+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+ }
+}
+
+export function makeSchedule(pairs, seed) {
+ const random = mulberry32(seed)
+ const measured = Array.from({ length: pairs }, (_, index) => (index < pairs / 2 ? 'AB' : 'BA'))
+ for (let index = measured.length - 1; index > 0; index--) {
+ const swap = Math.floor(random() * (index + 1))
+ ;[measured[index], measured[swap]] = [measured[swap], measured[index]]
+ }
+ return { seed, warmups: [...WARMUP_ORDERS], measured }
+}
+
+export function median(values) {
+ if (!values.length) throw new Error('median of no values')
+ const sorted = [...values].sort((a, b) => a - b), middle = sorted.length >> 1
+ return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2
+}
+
+const round = value => Math.round(value * 10) / 10
+
+export function bootstrapMedianCI(values, seed, resamples = BOOTSTRAP_RESAMPLES) {
+ const random = mulberry32(seed)
+ const medians = Array.from({ length: resamples }, () => median(values.map(() => values[Math.floor(random() * values.length)])))
+ medians.sort((a, b) => a - b)
+ return [medians[Math.floor(0.025 * resamples)], medians[Math.ceil(0.975 * resamples) - 1]]
+}
+
+// Measured samples only: warmups are excluded by flag, and an incomplete pair fails instead of dropping out.
+export function summarize(samples, seed) {
+ const measured = samples.filter(sample => !sample.warmup)
+ const pairs = new Map()
+ for (const sample of measured) {
+ const entry = pairs.get(sample.pair) ?? { pair: sample.pair, order: sample.order, A: null, B: null }
+ if (entry.order !== sample.order || entry[sample.variant] !== null) throw new Error(`pair ${sample.pair} is inconsistent`)
+ entry[sample.variant] = sample
+ pairs.set(sample.pair, entry)
+ }
+ const rows = [...pairs.values()].sort((a, b) => a.pair - b.pair).map(entry => {
+ if (!entry.A || !entry.B) throw new Error(`pair ${entry.pair} is incomplete`)
+ const raw = entry.B.imageReadyMs - entry.A.imageReadyMs
+ return { pair: entry.pair, order: entry.order, raw, deltaMs: round(raw) }
+ })
+ if (rows.length < 2) throw new Error('at least two measured pairs are required')
+ const deltas = rows.map(row => row.raw)
+ const variant = letter => {
+ const values = measured.filter(sample => sample.variant === letter)
+ return { n: values.length, medianImageReadyMs: round(median(values.map(sample => sample.imageReadyMs))), medianReadyMs: round(median(values.map(sample => sample.readyMs))) }
+ }
+ const [low, high] = bootstrapMedianCI(deltas, seed ^ 0x9e3779b9)
+ const byOrder = order => {
+ const values = rows.filter(row => row.order === order).map(row => row.raw)
+ return values.length ? round(median(values)) : null
+ }
+ return {
+ n: rows.length, A: variant('A'), B: variant('B'),
+ pairedDeltaMs: { definition: 'B - A per pair, imageReady', median: round(median(deltas)), ci95: [round(low), round(high)], method: `percentile bootstrap of the median, ${BOOTSTRAP_RESAMPLES} resamples` },
+ byOrder: { AB: byOrder('AB'), BA: byOrder('BA') },
+ pairs: rows.map(({ pair, order, deltaMs }) => ({ pair, order, deltaMs })),
+ }
+}
+
+// The host reads only the tab output line; the page itself has already timed everything it reports.
+export function parseTabOutput(stdout, expectedWindow) {
+ const text = stdout.replace(/\n$/, '')
+ if (text === 'NOTFOUND') throw new Error('the owned test tab was not found')
+ if (text.startsWith('ERROR:')) throw new Error(`AppleEvents error ${text.slice(0, 200)}`)
+ const match = /^WINDOW:(\d+)\|/.exec(text)
+ if (!match) throw new Error('unexpected AppleEvents output')
+ if (expectedWindow && match[1] !== expectedWindow) throw new Error(`the tab is in window ${match[1]}, expected ${expectedWindow}`)
+ return text.slice(match[0].length)
+}
+
+export function checkOrigin({ protocol, hostname }) {
+ if (protocol !== 'https:' || !hostname.startsWith('fx0-') || !hostname.endsWith('.labs.vercel.dev')) {
+ throw new Error(`the owned tab is not an fx0 preview origin (${protocol} ${hostname})`)
+ }
+}
+
+export function checkProbe(probe, session, revision = null) {
+ if (!probe?.ok) throw new Error(`source probe failed: ${probe?.error ?? 'no result'}`)
+ if (probe.status !== 200 || probe.type !== 'basic' || probe.redirected) throw new Error(`source probe answered ${probe.status} (${probe.type})`)
+ if (!String(probe.contentType).startsWith('application/json')) throw new Error('source probe is not JSON')
+ if (probe.protocol !== 1 || probe.sessionId !== session) throw new Error('source probe has the wrong protocol or session')
+ if (!Number.isSafeInteger(probe.revision) || probe.revision < 1 || (revision !== null && probe.revision !== revision)) throw new Error('source probe revision changed')
+ if (!(probe.sourceLength > 0) || !/^[0-9a-f]{64}$/.test(probe.sha256 ?? '')) throw new Error('source probe has no source hash')
+ if (probe.images !== 0) throw new Error('the live source has images; this benchmark requires none')
+ return probe
+}
+
+export function checkResources(entries, variant, session) {
+ const formats = entries.map(entry => entry.format)
+ const expected = variant === 'A' ? ['json', 'html'] : ['html']
+ if (JSON.stringify(formats) !== JSON.stringify(expected)) throw new Error(`variant ${variant} requests were ${formats.join(',') || 'none'}`)
+ if (entries.some(entry => entry.session !== session)) throw new Error('a frame request has the wrong session')
+}
+
+export function checkTrial(trial, { variant, session, revision, viewport }) {
+ if (!trial?.ok) throw Object.assign(new Error(`trial ${variant} failed: ${trial?.error ?? 'no result'}`), { diagnostics: trial ?? null })
+ if (trial.errors !== 0) throw new Error(`trial ${variant} observed hydration or runtime errors`)
+ if (trial.paint.sessionId !== session || trial.paint.revision !== revision || trial.paint.protocol !== 1 || typeof trial.paint.updateReady !== 'boolean') {
+ throw new Error(`trial ${variant} painted the wrong revision or protocol`)
+ }
+ for (const side of [trial.viewport.before, trial.viewport.after]) {
+ if (side.visibility !== 'visible') throw new Error('the owned tab was hidden during a trial')
+ if (side.width !== viewport.width || side.height !== viewport.height) throw new Error('the viewport changed during a trial')
+ }
+ if (!(trial.readyMs >= 0) || trial.imageReadyMs < trial.readyMs) throw new Error(`trial ${variant} has inconsistent timings`)
+ if (trial.frame.width !== FRAME.width || trial.frame.height !== FRAME.height) throw new Error('the front frame is not 1000x720')
+ if (trial.reply.error) throw new Error(`capture failed: ${trial.reply.error}`)
+ const { blobs } = trial.reply
+ if (blobs.length < 1 || blobs.length > MAX_CAPTURE_TILES) throw new Error(`capture returned ${blobs.length} images`)
+ if (blobs.reduce((sum, blob) => sum + blob.size, 0) > MAX_CAPTURE_BYTES) throw new Error('capture images contain too many bytes in total')
+ for (const blob of blobs) {
+ if (!IMAGE_TYPES.includes(blob.type) || !(blob.size > 0) || blob.size > MAX_CAPTURE_BYTES) throw new Error(`capture image is ${blob.type} ${blob.size} bytes`)
+ if (blob.width !== FRAME.width || blob.height !== FRAME.height) throw new Error('a captured image is not 1000x720')
+ }
+ checkResources(trial.resources, variant, session)
+ return trial
+}
+
+export function stripScripts(html) {
+ const stripped = html.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
+ if (/<script\b/i.test(stripped)) throw new Error('a script tag survived stripping')
+ return stripped
+}
+
+// Every browser step is a short expression; the only long text (the bundle) travels through a temp file.
+const INFO_SOURCE = 'JSON.stringify({ protocol: location.protocol, hostname: location.hostname, visibility: document.visibilityState, width: innerWidth, height: innerHeight })'
+const READ_PROBE = 'JSON.stringify(window.__abProbe ?? null)'
+const READ_TRIAL = 'JSON.stringify(window.__abTrial ?? null)'
+const READY = `JSON.stringify({
+ fixtureReady: window.fixtureReady === true,
+ frames: document.querySelectorAll('#preview-pane iframe').length,
+ frameRequests: performance.getEntriesByType('resource').filter(entry => new URL(entry.name).pathname === '/api/artifact-frame').length,
+ errors: window.observed ? observed.errors.length : -1,
+ api: typeof window.setRevision === 'function' && typeof window.captureFrame === 'function' && typeof window.unmountFixture === 'function',
+})`
+
+// Read-only probe: the same authenticated GET the app makes; the hash is computed in the page.
+function probeSource(session) {
+ return `(() => {
+ window.__abProbe = null
+ ;(async () => {
+ try {
+ const response = await fetch(${JSON.stringify(`/api/artifact-frame?session=${encodeURIComponent(session)}&format=json`)}, { cache: 'no-store', redirect: 'manual', credentials: 'same-origin' })
+ const contentType = response.headers.get('content-type') || ''
+ const body = response.status === 200 && contentType.startsWith('application/json') ? JSON.parse(await response.text()) : null
+ const html = typeof body?.sourceHtml === 'string' ? body.sourceHtml : ''
+ const digest = html ? [...new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(html)))].map(byte => byte.toString(16).padStart(2, '0')).join('') : null
+ window.__abProbe = { ok: true, status: response.status, type: response.type, redirected: response.redirected, contentType,
+ protocol: body?.protocol ?? null, sessionId: body?.sessionId ?? null, revision: body?.revision ?? null,
+ sourceLength: html.length, images: Array.isArray(body?.images) ? body.images.length : null, sha256: digest }
+ } catch (error) { window.__abProbe = { ok: false, error: String(error && error.message || error) } }
+ })()
+ return 'started'
+})()`
+}
+
+function resetSource(html) {
+ return `(() => {
+ performance.clearResourceTimings()
+ window.__abTrial = null
+ if (typeof window.unmountFixture === 'function') window.unmountFixture()
+ window.fixtureReady = false
+ delete window.setRevision
+ delete window.captureFrame
+ delete window.unmountFixture
+ document.open(); document.write(${JSON.stringify(html)}); document.close()
+ return 'reset'
+})()`
+}
+
+// The measured interval runs in the page: setRevision to a decoded image. The host only polls for the result.
+function trialSource({ session, revision, expectedRequests }) {
+ return `(() => {
+ const session = ${JSON.stringify(session)}, revision = ${revision}, expectedRequests = ${expectedRequests}
+ const frameSelector = '#preview-pane iframe[title="Design artifact preview"]:not([aria-hidden="true"])'
+ const round = value => Math.round(value * 1000) / 1000
+ const viewport = () => ({ width: innerWidth, height: innerHeight, visibility: document.visibilityState })
+ const tick = () => new Promise(resolve => { requestAnimationFrame(() => resolve()); setTimeout(resolve, 100) })
+ const waitFor = async (label, check) => {
+ const deadline = performance.now() + 15000
+ for (;;) {
+ const value = check()
+ if (value) return value
+ if (performance.now() > deadline) throw new Error('timed out waiting for ' + label)
+ await tick()
+ }
+ }
+ window.__abTrial = null
+ const diagnostics = { phase: 'waiting for paint' }
+ ;(async () => {
+ try {
+ const before = viewport()
+ if (before.visibility !== 'visible') throw new Error('the tab is not visible')
+ performance.clearResourceTimings()
+ const start = performance.now()
+ window.setRevision(revision)
+ const paint = await waitFor('the trusted paint', () => observed.paints.find(entry => entry.observedAt >= start
+ && entry.sessionId === session && entry.revision === revision && entry.protocol === 1 && typeof entry.updateReady === 'boolean'))
+ const front = await waitFor('the front frame', () => {
+ const frame = document.querySelector(frameSelector)
+ return frame && frame.getAttribute('src') === paint.mountUrl ? frame : null
+ })
+ const readyMs = paint.observedAt - start
+ Object.assign(diagnostics, { phase: 'capturing image', readyMs })
+ const reply = await window.captureFrame(front.contentWindow, crypto.randomUUID(), false)
+ const imageReadyMs = performance.now() - start
+ diagnostics.imageReadyMs = imageReadyMs
+ if (!reply) throw new Error('the capture reply did not arrive')
+ const blobs = []
+ for (const blob of reply.blobs) {
+ const bitmap = await createImageBitmap(blob)
+ blobs.push({ type: blob.type, size: blob.size, width: bitmap.width, height: bitmap.height })
+ bitmap.close()
+ }
+ const frameEntries = () => performance.getEntriesByType('resource')
+ .filter(entry => new URL(entry.name).pathname === '/api/artifact-frame')
+ .sort((a, b) => a.startTime - b.startTime)
+ .map(entry => {
+ const url = new URL(entry.name)
+ return { format: url.searchParams.get('format') === 'json' ? 'json' : 'html', session: url.searchParams.get('session'),
+ initiator: entry.initiatorType, startMs: round(entry.startTime - start), durationMs: round(entry.duration),
+ ttfbMs: round(entry.responseStart - entry.requestStart), transferSize: entry.transferSize,
+ decodedBodySize: entry.decodedBodySize, responseStatus: entry.responseStatus }
+ })
+ Object.assign(diagnostics, { phase: 'collecting request timings', images: blobs, frameTiming: reply.timing ?? null })
+ const resources = await waitFor('the frame request entries', () => { const entries = frameEntries(); return entries.length >= expectedRequests ? entries : null })
+ const rect = front.getBoundingClientRect()
+ window.__abTrial = { ok: true, readyMs: round(readyMs), imageReadyMs: round(imageReadyMs),
+ paint: { sessionId: paint.sessionId, revision: paint.revision, protocol: paint.protocol, updateReady: paint.updateReady },
+ mountId: new URL(paint.mountUrl, location.origin).searchParams.get('mount'),
+ frame: { width: Math.round(rect.width), height: Math.round(rect.height) },
+ reply: { error: reply.error ?? null, truncated: reply.truncated === true, blobs, timing: reply.timing ?? null },
+ resources, viewport: { before, after: viewport() }, errors: observed.errors.length }
+ } catch (error) {
+ diagnostics.resources = performance.getEntriesByType('resource').map(entry => {
+ const url = new URL(entry.name, location.origin)
+ return { path: url.pathname, format: url.searchParams.get('format'), initiator: entry.initiatorType,
+ durationMs: entry.duration, startTime: entry.startTime }
+ })
+ try {
+ const frameWindow = document.querySelector(frameSelector)?.contentWindow
+ diagnostics.frameNavigation = frameWindow ? frameWindow.performance.getEntriesByType('navigation').map(entry => ({
+ path: new URL(entry.name).pathname, startTime: entry.startTime, durationMs: entry.duration,
+ responseStatus: entry.responseStatus, timeOrigin: frameWindow.performance.timeOrigin,
+ })) : []
+ } catch { diagnostics.frameNavigationError = 'Frame navigation timing is not accessible' }
+ window.__abTrial = { ok: false, error: String(error && error.message || error), diagnostics }
+ }
+ })()
+ return 'started'
+})()`
+}
+
+// Exported for compile-only tests; the runner itself never evaluates these outside the tab.
+export const browserSources = { info: INFO_SOURCE, ready: READY, readProbe: READ_PROBE, readTrial: READ_TRIAL, probe: probeSource, reset: resetSource, trial: trialSource }
+
+// AppleEvents execution has different resource-timing visibility from page scripts.
+// Run trusted harness code in the page and return only its result through the shared DOM.
+export function pageBridge(source) {
+ const attribute = 'data-fx0-ab-result'
+ const pageScript = `try {
+ const value = (0, eval)(${JSON.stringify(source)});
+ document.documentElement.setAttribute('${attribute}', JSON.stringify({ ok: true, value }));
+ } catch (error) {
+ document.documentElement.setAttribute('${attribute}', JSON.stringify({ ok: false, error: String(error.message || error) }));
+ }`
+ return `(() => {
+ document.documentElement.removeAttribute('${attribute}');
+ const script = document.createElement('script');
+ script.textContent = ${JSON.stringify(pageScript)};
+ document.head.append(script);
+ script.remove();
+ const text = document.documentElement.getAttribute('${attribute}');
+ if (text === null) throw new Error('Page script did not execute; check the page CSP');
+ const result = JSON.parse(text);
+ if (!result.ok) throw new Error(result.error);
+ return result.value === undefined ? '' : String(result.value);
+ })()`
+}
+
+async function runTab(ctx, source) {
+ const file = join(ctx.dir, `payload-${ctx.count++}.js`)
+ await writeFile(file, pageBridge(source))
+ try {
+ const { stdout } = await execFileAsync('osascript', [ctx.script, ctx.tab, file, ctx.window ?? ''], { timeout: TAB_TIMEOUT_MS, maxBuffer: 1024 * 1024 })
+ return parseTabOutput(stdout, ctx.window)
+ } finally { await rm(file, { force: true }) }
+}
+
+async function pollUntil(ctx, source, done, label, timeoutMs = 20_000) {
+ const deadline = Date.now() + timeoutMs
+ let last
+ do {
+ last = JSON.parse(await runTab(ctx, source))
+ if (done(last)) return last
+ await sleep(100)
+ } while (Date.now() < deadline)
+ throw new Error(`timed out: ${label}; last state ${JSON.stringify(last).slice(0, 300)}`)
+}
+
+async function probe(ctx, session) {
+ await runTab(ctx, probeSource(session))
+ return pollUntil(ctx, READ_PROBE, value => value !== null, 'the source probe')
+}
+
+function assertSource(current, expected) {
+ if (current.sha256 !== expected.sha256 || current.sourceLength !== expected.sourceLength) throw new Error('the live source changed during the run')
+}
+
+async function preflight(ctx, session) {
+ // Let Chrome process the owned window's bounds before freezing its viewport.
+ await runTab(ctx, INFO_SOURCE)
+ const info = JSON.parse(await runTab(ctx, INFO_SOURCE))
+ checkOrigin(info)
+ if (info.visibility !== 'visible') throw new Error('the owned tab is not visible')
+ const first = checkProbe(await probe(ctx, session), session)
+ return {
+ origin: `https://${info.hostname}`, viewport: { width: info.width, height: info.height },
+ source: { revision: first.revision, sha256: first.sha256, sourceLength: first.sourceLength, images: first.images },
+ }
+}
+
+async function buildVariant(dir, session, experiment) {
+ await mkdir(dir, { recursive: true })
+ const { bundle, html } = await buildPaneFixture(dir, experiment, { sessionId: session, initialRevision: 0 })
+ return { html: stripScripts(html), observer: `${OBSERVER}\n;'observer'`, bundle: `${bundle.toString('utf8')}\n;'bundle'` }
+}
+
+async function runSample(ctx, run, letter, meta) {
+ const variant = run.variants[letter]
+ const before = checkProbe(await probe(ctx, run.session), run.session, run.source.revision)
+ assertSource(before, run.source)
+ if ((await runTab(ctx, resetSource(variant.html))) !== 'reset') throw new Error('the fixture reset did not complete')
+ await runTab(ctx, variant.observer)
+ await runTab(ctx, variant.bundle)
+ await pollUntil(ctx, READY, ready => ready.fixtureReady && ready.frames === 0 && ready.frameRequests === 0 && ready.errors === 0 && ready.api,
+ 'a fixture ready without frames or requests')
+ await runTab(ctx, trialSource({ session: run.session, revision: run.source.revision, expectedRequests: letter === 'A' ? 2 : 1 }))
+ const rawTrial = await pollUntil(ctx, READ_TRIAL, value => value !== null, 'the measured trial', 30_000)
+ let trial
+ try {
+ trial = checkTrial(rawTrial, { variant: letter, session: run.session, revision: run.source.revision, viewport: run.preflight.viewport })
+ } catch (error) {
+ error.diagnostics ??= rawTrial
+ throw error
+ }
+ const after = checkProbe(await probe(ctx, run.session), run.session, run.source.revision)
+ assertSource(after, run.source)
+ run.mountIds.add(trial.mountId)
+ return {
+ ...meta, variant: letter, readyMs: trial.readyMs, imageReadyMs: trial.imageReadyMs,
+ paintUpdateReady: trial.paint.updateReady, truncated: trial.reply.truncated, frameTiming: trial.reply.timing,
+ blobs: trial.reply.blobs.map(({ type, size, width, height }) => ({ type, size, width, height })),
+ resources: trial.resources.map(({ format, startMs, durationMs, ttfbMs, transferSize, decodedBodySize, responseStatus, initiator }) => ({ format, startMs, durationMs, ttfbMs, transferSize, decodedBodySize, responseStatus, initiator })),
+ source: { before: { revision: before.revision, sha256: before.sha256 }, after: { revision: after.revision, sha256: after.sha256 } },
+ }
+}
+
+async function main(options) {
+ const dir = await mkdtemp(join(tmpdir(), 'fx0-live-ab-'))
+ const ctx = { dir, tab: options.tab, window: options.window, count: 0, script: join(dir, 'tab.applescript') }
+ await writeFile(ctx.script, TAB_SCRIPT)
+ const plan = makeSchedule(options.pairs, options.seed)
+ const result = { kind: 'fx0-preview-live-ab', startedAt: new Date().toISOString(), seed: options.seed, schedule: plan,
+ windowBounds: [0, 24, 1440, 1000],
+ variants: { A: 'reconstructed client flow: helper LEGACY_BASELINE removed, not the deployed old bundle', B: 'current unmodified PreviewPane' },
+ limits: ['one owned tab, one revision, one network path', 'the reused Next page timers may keep running after document.open',
+ 'the HTTP cache is default and warm after warmups; cold is not claimed', 'imageReady excludes publication, LLM and upload work',
+ 'n=30 pairs is a small sample; the interval is a percentile bootstrap of the median'],
+ samples: [], completed: false }
+ try {
+ result.preflight = await preflight(ctx, options.session)
+ const run = { session: options.session, preflight: result.preflight, source: result.preflight.source, mountIds: new Set(),
+ variants: { A: await buildVariant(join(dir, 'A'), options.session, LEGACY_BASELINE), B: await buildVariant(join(dir, 'B'), options.session, null) } }
+ const orders = [...plan.warmups.map((order, index) => ({ order, pair: `warmup-${index + 1}`, warmup: true })),
+ ...plan.measured.map((order, index) => ({ order, pair: index + 1, warmup: false }))]
+ for (const { order, pair, warmup } of orders) {
+ for (const letter of order) {
+ result.samples.push(await runSample(ctx, run, letter, { index: result.samples.length + 1, pair, order, warmup }))
+ }
+ console.log(`${warmup ? 'Warmup' : 'Pair'} ${pair}: ${order} completed`)
+ }
+ if (run.mountIds.size !== result.samples.length) throw new Error('frame mount URLs were not unique per sample')
+ result.summary = summarize(result.samples, options.seed)
+ result.completed = true
+ } catch (error) {
+ result.error = error.message
+ result.failure = error.diagnostics ?? null
+ process.exitCode = 1
+ } finally {
+ result.finishedAt = new Date().toISOString()
+ await writeFile(options.out, `${JSON.stringify(result, null, 2)}\n`, { flag: 'wx' })
+ await rm(dir, { recursive: true, force: true })
+ }
+ return result
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+ try {
+ const options = parseArgs(process.argv.slice(2))
+ if (existsSync(options.out)) throw new Error(`${options.out} already exists; choose a new --out`)
+ const result = await main(options)
+ console.log(JSON.stringify({ completed: result.completed, error: result.error ?? null, summary: result.summary ?? null }, null, 2))
+ } catch (error) {
+ console.error(error.message)
+ process.exitCode = 2
+ }
+}
diff --git a/tests/preview-live-ab.test.mjs b/tests/preview-live-ab.test.mjs
new file mode 100644
index 0000000..47b7ef9
--- /dev/null
+++ b/tests/preview-live-ab.test.mjs
@@ -0,0 +1,209 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import vm from 'node:vm'
+import {
+ bootstrapMedianCI, browserSources, checkOrigin, checkProbe, checkResources, checkTrial, makeSchedule,
+ median, pageBridge, parseArgs, parseTabOutput, stripScripts, summarize,
+} from './preview-live-ab.mjs'
+
+const SESSION = '11111111-1111-4111-8111-111111111111'
+const ARGS = ['--tab', '1267394049', '--session', SESSION, '--out', '/tmp/x.json']
+const VIEWPORT = { width: 1440, height: 900 }
+const SHA = 'a'.repeat(64)
+
+function trial(overrides = {}) {
+ return {
+ ok: true, readyMs: 40, imageReadyMs: 120, errors: 0, mountId: 'm-1',
+ paint: { sessionId: SESSION, revision: 2, protocol: 1, updateReady: false },
+ frame: { width: 1000, height: 720 },
+ reply: { error: null, truncated: false, timing: null, blobs: [{ type: 'image/webp', size: 1000, width: 1000, height: 720 }] },
+ resources: [{ format: 'html', session: SESSION }],
+ viewport: { before: { ...VIEWPORT, visibility: 'visible' }, after: { ...VIEWPORT, visibility: 'visible' } },
+ ...overrides,
+ }
+}
+
+function sample(pair, variant, order, imageReadyMs, warmup = false) {
+ return { pair, variant, order, warmup, readyMs: imageReadyMs / 2, imageReadyMs }
+}
+
+test('parseArgs accepts the documented CLI and rejects bad inputs before any AppleEvents', () => {
+ const options = parseArgs([...ARGS, '--pairs', '30', '--window', '1267393606', '--seed', '7'])
+ assert.deepEqual(options, { tab: '1267394049', window: '1267393606', session: SESSION, pairs: 30, seed: 7, out: '/tmp/x.json' })
+ assert.equal(Number.isInteger(parseArgs(ARGS).seed), true)
+ assert.throws(() => parseArgs([...ARGS, '--pairs', '29']), /even integer/)
+ assert.throws(() => parseArgs([...ARGS, '--pairs', '0']), /even integer/)
+ assert.throws(() => parseArgs([...ARGS, '--pairs', '202']), /even integer/)
+ assert.throws(() => parseArgs(['--tab', '12a', '--session', SESSION, '--out', '/tmp/x']), /numeric tab id/)
+ assert.throws(() => parseArgs(['--tab', '1', '--session', 'not-a-uuid', '--out', '/tmp/x']), /session UUID/)
+ assert.throws(() => parseArgs(['--tab', '1', '--session', SESSION]), /--out is required/)
+ assert.throws(() => parseArgs([...ARGS, '--bogus', '1']), /unknown option/)
+ assert.throws(() => parseArgs(['--tab']), /missing value/)
+ assert.throws(() => parseArgs([...ARGS, '--seed', '-1']), /uint32/)
+})
+
+test('makeSchedule is seeded, balanced 15 AB and 15 BA, and fixes two warmup pairs outside the measured set', () => {
+ const schedule = makeSchedule(30, 7)
+ assert.deepEqual(schedule.warmups, ['AB', 'BA'])
+ assert.equal(schedule.measured.length, 30)
+ assert.equal(schedule.measured.filter(order => order === 'AB').length, 15)
+ assert.equal(schedule.measured.filter(order => order === 'BA').length, 15)
+ assert.deepEqual(makeSchedule(30, 7).measured, schedule.measured)
+ assert.notDeepEqual(makeSchedule(30, 8).measured, schedule.measured)
+})
+
+test('median and bootstrap are deterministic and bracket the median', () => {
+ assert.equal(median([3, 1, 2]), 2)
+ assert.equal(median([4, 1, 3, 2]), 2.5)
+ assert.throws(() => median([]), /no values/)
+ const deltas = [-5, -3, -2, 1, 2, 4, 6, 7, -1, 0]
+ const [low, high] = bootstrapMedianCI(deltas, 99)
+ assert.deepEqual(bootstrapMedianCI(deltas, 99), [low, high])
+ assert.ok(low <= median(deltas) && median(deltas) <= high)
+})
+
+test('summarize excludes warmups, pairs B - A, and refuses incomplete or missing pairs', () => {
+ const samples = [
+ sample(1, 'A', 'AB', 9999, true), sample(1, 'B', 'AB', 1, true),
+ sample(2, 'A', 'AB', 100), sample(2, 'B', 'AB', 80),
+ sample(3, 'B', 'BA', 70), sample(3, 'A', 'BA', 100),
+ sample(4, 'A', 'AB', 120), sample(4, 'B', 'AB', 90),
+ ]
+ const summary = summarize(samples, 5)
+ assert.equal(summary.n, 3)
+ assert.deepEqual(summary.pairs.map(pair => pair.deltaMs), [-20, -30, -30])
+ assert.equal(summary.A.n, 3)
+ assert.equal(summary.A.medianImageReadyMs, 100)
+ assert.equal(summary.B.medianImageReadyMs, 80)
+ assert.equal(summary.pairedDeltaMs.median, -30)
+ assert.equal(summary.byOrder.AB, -25)
+ assert.equal(summary.byOrder.BA, -30)
+ assert.ok(summary.pairedDeltaMs.ci95[0] <= -30 && -30 <= summary.pairedDeltaMs.ci95[1])
+ assert.throws(() => summarize([sample(1, 'A', 'AB', 1), sample(2, 'A', 'AB', 2), sample(2, 'B', 'AB', 2)], 1), /pair 1 is incomplete/)
+ assert.throws(() => summarize([sample(1, 'A', 'AB', 1), sample(1, 'B', 'AB', 2)], 1), /at least two measured pairs/)
+})
+
+test('parseTabOutput maps NOTFOUND, AppleEvents errors and window mismatches to failures', () => {
+ assert.throws(() => parseTabOutput('NOTFOUND\n'), /not found/)
+ assert.throws(() => parseTabOutput('ERROR:-2700:Turned off\n'), /AppleEvents error/)
+ assert.throws(() => parseTabOutput('WINDOW:111|"ok"\n', '222'), /expected 222/)
+ assert.throws(() => parseTabOutput('whatever'), /unexpected/)
+ assert.equal(parseTabOutput('WINDOW:111|{"a":"x|y"}\n', '111'), '{"a":"x|y"}')
+})
+
+test('checkOrigin accepts only https fx0 preview hosts under labs.vercel.dev', () => {
+ checkOrigin({ protocol: 'https:', hostname: 'fx0-git-esteban-fx0-preview-first-load.labs.vercel.dev' })
+ for (const origin of [
+ { protocol: 'http:', hostname: 'fx0-a.labs.vercel.dev' },
+ { protocol: 'https:', hostname: 'evil.labs.vercel.dev' },
+ { protocol: 'https:', hostname: 'fx0-a.example.com' },
+ ]) assert.throws(() => checkOrigin(origin), /not an fx0 preview origin/)
+})
+
+test('checkProbe requires an unredirected 200 JSON protocol-1 probe for the session with images 0', () => {
+ const good = { ok: true, status: 200, type: 'basic', redirected: false, contentType: 'application/json; charset=utf-8', protocol: 1, sessionId: SESSION, revision: 2, sourceLength: 1506, images: 0, sha256: SHA }
+ assert.equal(checkProbe(good, SESSION, 2), good)
+ assert.throws(() => checkProbe({ ...good, status: 401 }, SESSION), /answered 401/)
+ assert.throws(() => checkProbe({ ...good, redirected: true }, SESSION), /answered 200/)
+ assert.throws(() => checkProbe({ ...good, contentType: 'text/html' }, SESSION), /not JSON/)
+ assert.throws(() => checkProbe({ ...good, sessionId: 'other' }, SESSION), /protocol or session/)
+ assert.throws(() => checkProbe({ ...good, revision: 3 }, SESSION, 2), /revision changed/)
+ assert.throws(() => checkProbe({ ...good, revision: 0 }, SESSION), /revision changed/)
+ assert.throws(() => checkProbe({ ...good, images: 1 }, SESSION), /requires none/)
+ assert.throws(() => checkProbe({ ok: false, error: 'network' }, SESSION), /probe failed/)
+})
+
+test('checkResources enforces the JSON-first order for A and the direct HTML load for B', () => {
+ checkResources([{ format: 'json', session: SESSION }, { format: 'html', session: SESSION }], 'A', SESSION)
+ checkResources([{ format: 'html', session: SESSION }], 'B', SESSION)
+ assert.throws(() => checkResources([{ format: 'html', session: SESSION }], 'A', SESSION), /requests were html/)
+ assert.throws(() => checkResources([{ format: 'json', session: SESSION }, { format: 'html', session: SESSION }], 'B', SESSION), /requests were json,html/)
+ assert.throws(() => checkResources([{ format: 'html', session: 'x' }], 'B', SESSION), /wrong session/)
+})
+
+test('checkTrial accepts a clean trial and rejects each broken invariant', () => {
+ const options = { variant: 'B', session: SESSION, revision: 2, viewport: VIEWPORT }
+ assert.equal(checkTrial(trial(), options).imageReadyMs, 120)
+ assert.throws(() => checkTrial(trial({ ok: false, error: 'boom' }), options), /failed: boom/)
+ assert.throws(() => checkTrial(trial({ errors: 1 }), options), /hydration or runtime errors/)
+ assert.throws(() => checkTrial(trial({ paint: { sessionId: SESSION, revision: 1, protocol: 1, updateReady: false } }), options), /wrong revision/)
+ assert.throws(() => checkTrial(trial({ viewport: { before: { ...VIEWPORT, visibility: 'hidden' }, after: { ...VIEWPORT, visibility: 'visible' } } }), options), /hidden/)
+ assert.throws(() => checkTrial(trial({ viewport: { before: { width: 800, height: 900, visibility: 'visible' }, after: { width: 800, height: 900, visibility: 'visible' } } }), options), /viewport changed/)
+ assert.throws(() => checkTrial(trial({ reply: { error: 'too-large', truncated: false, blobs: [] } }), options), /capture failed/)
+ assert.throws(() => checkTrial(trial({ reply: { error: null, truncated: false, blobs: [] } }), options), /0 images/)
+ assert.throws(() => checkTrial(trial({ reply: { error: null, truncated: false, blobs: Array(9).fill({ type: 'image/png', size: 1, width: 1000, height: 720 }) } }), options), /9 images/)
+ assert.throws(() => checkTrial(trial({ reply: { error: null, truncated: false, blobs: [{ type: 'text/html', size: 10, width: 1000, height: 720 }] } }), options), /text\/html/)
+ assert.throws(() => checkTrial(trial({ reply: { error: null, truncated: false, blobs: [{ type: 'image/png', size: 601 * 1024, width: 1000, height: 720 }] } }), options), /bytes/)
+ assert.throws(() => checkTrial(trial({ reply: { error: null, truncated: false, blobs: [{ type: 'image/png', size: 1, width: 1280, height: 800 }] } }), options), /not 1000x720/)
+ assert.throws(() => checkTrial(trial({ readyMs: 130 }), options), /inconsistent timings/)
+ assert.throws(() => checkTrial(trial({ resources: [] }), { ...options, variant: 'A' }), /requests were none/)
+})
+
+test('stripScripts removes every script tag and keeps the markup the fixture needs', () => {
+ const html = '<div id="app"><p>x</p></div><script>window.observed={}</script><script defer src="/bundle.js"></script><style>p{}</style>'
+ assert.equal(stripScripts(html), '<div id="app"><p>x</p></div><style>p{}</style>')
+})
+
+test('a missing parent resource entry preserves the image, partial clocks and frame navigation diagnosis', async () => {
+ let now = 10
+ const mountUrl = `/api/artifact-frame?session=${SESSION}&v=2&mount=test`
+ const location = { origin: 'https://fx0-test.labs.vercel.dev' }
+ const navigation = { name: location.origin + mountUrl, startTime: 0, duration: 80, responseStatus: 200 }
+ const front = { getAttribute: () => mountUrl,
+ contentWindow: { performance: { timeOrigin: 1000, getEntriesByType: () => [navigation] } } }
+ const observed = { paints: [], errors: [] }
+ const window = {
+ setRevision: () => observed.paints.push({ observedAt: now, sessionId: SESSION, revision: 2, protocol: 1, updateReady: true, mountUrl }),
+ captureFrame: async () => ({ blobs: [{ type: 'image/png', size: 30 }], timing: { fontsMs: 0, libraryMs: 2, encodeMs: 1 } }),
+ }
+ vm.runInNewContext(browserSources.trial({ session: SESSION, revision: 2, expectedRequests: 2 }), {
+ window, observed, location, URL, innerWidth: 1440, innerHeight: 900,
+ document: { visibilityState: 'visible', querySelector: () => front },
+ crypto: { randomUUID: () => 'capture-test' },
+ performance: { now: () => now++, clearResourceTimings() {}, getEntriesByType: () => [{
+ name: location.origin + `/api/artifact-frame?session=${SESSION}&format=json`, initiatorType: 'fetch', duration: 50, startTime: 1,
+ }] },
+ createImageBitmap: async () => ({ width: 1000, height: 720, close() {} }),
+ requestAnimationFrame: callback => { now += 20_000; callback() }, setTimeout() {},
+ })
+ await new Promise(resolve => setImmediate(resolve))
+ const failed = window.__abTrial
+ assert.equal(failed.ok, false)
+ assert.match(failed.error, /frame request entries/)
+ assert.equal(failed.diagnostics.phase, 'collecting request timings')
+ assert.ok(failed.diagnostics.imageReadyMs >= failed.diagnostics.readyMs)
+ assert.equal(failed.diagnostics.images[0].size, 30)
+ assert.equal(failed.diagnostics.resources.length, 1)
+ assert.equal(failed.diagnostics.resources[0].format, 'json')
+ assert.equal(failed.diagnostics.frameNavigation[0].responseStatus, 200)
+ assert.throws(() => checkTrial(failed, { variant: 'A' }), error => error.diagnostics === failed)
+})
+
+test('the DOM bridge executes in the page realm rather than the AppleEvents realm', () => {
+ const attributes = new Map()
+ const document = {
+ documentElement: { setAttribute: (name, value) => attributes.set(name, value), getAttribute: name => attributes.get(name) ?? null, removeAttribute: name => attributes.delete(name) },
+ createElement: () => ({ textContent: '', remove() {} }), head: {},
+ }
+ const page = vm.createContext({ window: {}, document })
+ document.head.append = script => vm.runInContext(script.textContent, page)
+ const automation = { window: {}, document }
+ assert.equal(vm.runInNewContext(pageBridge('window.marker = "page"; "done"'), automation), 'done')
+ assert.equal(page.window.marker, 'page')
+ assert.equal(automation.window.marker, undefined)
+ assert.throws(() => vm.runInNewContext(pageBridge('throw new Error("page failure")'), automation), /page failure/)
+ document.head.append = () => {}
+ assert.throws(() => vm.runInNewContext(pageBridge('"blocked"'), automation), /Page script did not execute/)
+})
+
+test('browser payloads compile as JavaScript without running', () => {
+ const sources = {
+ info: browserSources.info, ready: browserSources.ready, readProbe: browserSources.readProbe, readTrial: browserSources.readTrial,
+ probe: browserSources.probe(SESSION), reset: browserSources.reset('<div id="app"></div>'),
+ trial: browserSources.trial({ session: SESSION, revision: 2, expectedRequests: 2 }),
+ }
+ for (const [name, source] of Object.entries(sources)) new vm.Script(source, { filename: name })
+ assert.match(sources.probe, /format=json/)
+ assert.match(sources.trial, /revision = 2,/)
+ assert.match(sources.trial, /window\.setRevision\(revision\)/)
+})
diff --git a/tests/preview-update.browser.mjs b/tests/preview-update.browser.mjs
index 7121d83..6d35252 100644
--- a/tests/preview-update.browser.mjs
+++ b/tests/preview-update.browser.mjs
@@ -1,11 +1,8 @@
import assert from 'node:assert/strict'
import test from 'node:test'
-import { execFile, spawn } from 'node:child_process'
+import { spawn } from 'node:child_process'
import { existsSync } from 'node:fs'
-import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'
-import { registerHooks } from 'node:module'
-import { fileURLToPath, pathToFileURL } from 'node:url'
-import { promisify } from 'node:util'
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import http from 'node:http'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -15,6 +12,7 @@ import { markRemoteImages } from '../lib/remote-image.ts'
import { CAPTURE_TIMING_HEADER } from '../lib/capture-protocol.ts'
import { DESIGN_CAPTURE_RELAY, designCaptureScript } from '../lib/design-capture.ts'
import { BLANK_IMAGE, DESIGN_ARTIFACT_EMBEDDED_CSP, MAX_EMBEDDED_IMAGE_BYTES, designFrameCsp, designFrameHtml, prepareDesignArtifactHtml, validateDesignArtifactHtml } from '../lib/design-artifact.ts'
+import { LEGACY_BASELINE, OBSERVER, buildPaneFixture } from './helpers/preview-pane-fixture.mjs'
// Opt-in, like design-artifact.browser.mjs. Node 24 supplies WebSocket and TS stripping.
// One Chrome and sequential cases keep this suite from spawning a browser per assertion.
@@ -24,13 +22,6 @@ const SESSION = 'preview-browser-session'
const TOKEN = 'preview-browser-ready'
// Opt-in experiment only (FX0_PREVIEW_PERF=1): it replaces the regular cases with a perf comparison.
const PERF = process.env.FX0_PREVIEW_PERF === '1'
-// Restore the old JSON-first behavior only in a temporary baseline compile. The candidate
-// runs the current production component unchanged; no benchmark switch enters production.
-const LEGACY_BASELINE = {
- marker: 'fx0-perf-baseline-json-first',
- from: 'if (!shown) {\n loadFull(desired);\n return;\n }',
- to: '/* fx0-perf-baseline-json-first */',
-}
const median = values => {
const sorted = values.filter(Number.isFinite).sort((a, b) => a - b), middle = sorted.length >> 1
return sorted.length === 0 ? null : sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2
@@ -86,28 +77,6 @@ function payload(revision, html, imageName) {
}
}
-// Observe real window messages without intercepting any production listener or send.
-const OBSERVER = `
-window.observed = { paints: [], results: [], boots: [], reports: [], errors: [], frameErrors: [] }
-addEventListener('message', event => {
- const frames = [...document.querySelectorAll('#active, #preview-pane iframe')], data = event.data
- if (!data || typeof data !== 'object') return
- const wrapper = frames.find(frame => event.source === frame.contentWindow)
- if (wrapper && event.origin === location.origin) {
- const entry = { ...data, mountUrl: wrapper.getAttribute('src'), observedAt: performance.now() }
- if (data.type === 'fx0-artifact-painted') observed.paints.push(entry)
- if (data.type === 'fx0-preview-result') observed.results.push(entry)
- if (data.type === 'fx0-preview-error') observed.frameErrors.push(entry)
- }
- const frame = frames.find(frame => event.source === frame.contentWindow.frames[0])
- if (frame && data.fixture === 'preview-update') {
- const entry = { ...data, mountUrl: frame.getAttribute('src') }
- if (data.kind === 'boot') observed.boots.push(entry)
- if (data.kind === 'state') { observed.reports.push(entry); if (observed.reports.length > 30) observed.reports.shift() }
- if (data.kind === 'error') observed.errors.push(data.error)
- }
-})`
-
const PARENT = `<!doctype html><html><head><title>Preview update browser test</title>
<style>html,body{margin:0}iframe{width:1000px;height:720px;border:0}</style></head><body><script>${OBSERVER}
window.mount = url => {
@@ -117,88 +86,6 @@ window.mount = url => {
}
</script></body></html>`
-async function buildPaneFixture(directory, experiment = null) {
- const output = await realpath(directory), root = fileURLToPath(new URL('../', import.meta.url))
- // Same narrow compile as html-ui.test.mjs, using this test's Node 24 executable.
- await promisify(execFile)(process.execPath, [join(root, 'node_modules/typescript/bin/tsc'),
- '--ignoreConfig', '--outDir', join(output, 'compiled'), '--strict', '--types', 'node',
- '--jsx', 'react-jsx', '--target', 'esnext', '--module', 'esnext',
- '--moduleResolution', 'bundler', '--skipLibCheck', '--esModuleInterop', 'components/preview-pane.tsx',
- ], { cwd: root, maxBuffer: 4 * 1024 * 1024 })
- if (experiment) {
- // Perf-only: remove the direct-load guard in the temporary legacy baseline, never in production.
- const file = join(output, 'compiled/components/preview-pane.js'), source = await readFile(file, 'utf8')
- assert.equal(source.split(experiment.from).length - 1, 1, 'the baseline matches exactly one compiled guard')
- await writeFile(file, source.replace(experiment.from, experiment.to))
- assert.equal((await readFile(file, 'utf8')).split(experiment.marker).length - 1, 1, 'the baseline marker is unique')
- }
- await writeFile(join(output, 'package.json'), '{"type":"module"}')
- // These controls change only the real component's props. No replacement hooks,
- // updater, transport or slot coordinator is bundled into the fixture.
- await writeFile(join(output, 'pane-fixture.js'), `
-import { createElement, useEffect, useState } from 'react'
-import PreviewPane from './compiled/components/preview-pane.js'
-import { signalAuthRequired } from './compiled/components/auth-ui.js'
-export function Fixture() {
- const [revision, setRevision] = useState(101), [viewing, setViewing] = useState(null)
- const [sessionId, setSession] = useState(${JSON.stringify(SESSION)}), [draft, setDraft] = useState(null)
- const [accessAllowed, setAccessAllowed] = useState(true), [capture, setCapture] = useState(null)
- useEffect(() => {
- window.captureResults = []
- window.authErrors = []
- Object.assign(window, { setRevision, setViewing, setSession, setDraft, setAccessAllowed, setCapture, closeAuthGate: signalAuthRequired, fixtureReady: true })
- const revoke = event => { window.authErrors.push(event.detail); setAccessAllowed(false); setRevision(0) }
- window.addEventListener('fx0:auth-required', revoke)
- return () => window.removeEventListener('fx0:auth-required', revoke)
- }, [])
- return createElement(PreviewPane, { source: { kind: 'design', sessionId, revision }, viewing, draft, accessAllowed, intro: false, onClose: () => setRevision(0),
- capture, onCaptureDone: (cid, answered) => { window.captureResults.push({ cid, answered, observedAt: performance.now() }); setCapture(current => current?.cid === cid ? null : current) },
- })
-}`)
- await writeFile(join(output, 'pane-entry.js'), `
-import { createElement } from 'react'
-import { hydrateRoot } from 'react-dom/client'
-import { Fixture } from './pane-fixture.js'
-hydrateRoot(document.getElementById('app'), createElement(Fixture), { onRecoverableError: error => observed.errors.push(error.message) })
-`)
- const { default: bundledWebpack } = await import('../node_modules/next/dist/compiled/webpack/webpack.js')
- const compiler = bundledWebpack.webpack({
- // Next's packaged webpack references a terser entry absent in this install.
- // Production React is still selected; minification is not under test here.
- mode: 'production', optimization: { minimize: false }, target: 'web', entry: join(output, 'pane-entry.js'),
- output: { path: join(output, 'browser'), filename: 'bundle.js' },
- module: { rules: [{ test: /\.js$/, resolve: { fullySpecified: false } }] },
- resolve: { modules: [join(root, 'node_modules')], extensions: ['.js'], extensionAlias: { '.ts': ['.js'] } },
- })
- try {
- const stats = await promisify(compiler.run.bind(compiler))()
- assert.equal(stats.hasErrors(), false, stats.toString({ all: false, errors: true }))
- } finally { await promisify(compiler.close.bind(compiler))() }
-
- const outputUrl = pathToFileURL(`${output}/`).href, projectUrl = new URL('../package.json', import.meta.url).href
- const hooks = registerHooks({ resolve(specifier, context, nextResolve) {
- if (!context.parentURL?.startsWith(outputUrl)) return nextResolve(specifier, context)
- if (specifier.startsWith('.')) return nextResolve(/\.js$/.test(specifier) ? specifier : `${specifier.replace(/\.ts$/, '')}.js`, context)
- return nextResolve(specifier, { ...context, parentURL: projectUrl })
- } })
- let markup
- try {
- const [{ Fixture }, { createElement }, { renderToString }] = await Promise.all([
- import(pathToFileURL(join(output, 'pane-fixture.js')).href), import('react'), import('react-dom/server'),
- ])
- markup = renderToString(createElement(Fixture))
- } finally { hooks.deregister() }
- return {
- bundle: await readFile(join(output, 'browser/bundle.js')),
- html: `<!doctype html><html><head><meta charset="utf-8"><title>Real PreviewPane</title><style>
-html,body,#app{margin:0;height:100%}#preview-pane{height:100%;display:flex;flex-direction:column}
-#preview-pane>div:last-of-type{position:relative;flex:1;min-height:720px}
-#preview-pane iframe{position:absolute;inset:0;width:1000px;height:720px;border:0}
-#preview-pane iframe[aria-hidden=true]{z-index:0;pointer-events:none}#preview-pane iframe:not([aria-hidden=true]){z-index:1}
-</style></head><body><script>${OBSERVER}</script><div id="app">${markup}</div><script defer src="/bundle.js"></script></body></html>`,
- }
-}
-
async function until(read, label, milliseconds = 8000) {
const deadline = Date.now() + milliseconds
let last